2024-11-27 07:24:13,688 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-27 07:24:13,711 main DEBUG Took 0.020562 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-27 07:24:13,712 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-27 07:24:13,712 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-27 07:24:13,714 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-27 07:24:13,715 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 07:24:13,724 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-27 07:24:13,740 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 07:24:13,742 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 07:24:13,743 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 07:24:13,743 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 07:24:13,744 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 07:24:13,745 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 07:24:13,746 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 07:24:13,747 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 07:24:13,748 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 07:24:13,748 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 07:24:13,749 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 07:24:13,750 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 07:24:13,751 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 07:24:13,751 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 07:24:13,752 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 07:24:13,753 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 07:24:13,753 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 07:24:13,754 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 07:24:13,754 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 07:24:13,755 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 07:24:13,755 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 07:24:13,756 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 07:24:13,757 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 07:24:13,757 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 07:24:13,758 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 07:24:13,758 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-27 07:24:13,760 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 07:24:13,762 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-27 07:24:13,764 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-27 07:24:13,765 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-27 07:24:13,767 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-27 07:24:13,767 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-27 07:24:13,777 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-27 07:24:13,780 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-27 07:24:13,782 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-27 07:24:13,782 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-27 07:24:13,783 main DEBUG createAppenders(={Console}) 2024-11-27 07:24:13,784 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-27 07:24:13,784 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-27 07:24:13,784 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-27 07:24:13,785 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-27 07:24:13,785 main DEBUG OutputStream closed 2024-11-27 07:24:13,786 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-27 07:24:13,786 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-27 07:24:13,786 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-27 07:24:13,873 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-27 07:24:13,876 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-27 07:24:13,878 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-27 07:24:13,879 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-27 07:24:13,881 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-27 07:24:13,881 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-27 07:24:13,882 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-27 07:24:13,882 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-27 07:24:13,883 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-27 07:24:13,883 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-27 07:24:13,884 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-27 07:24:13,884 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-27 07:24:13,885 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-27 07:24:13,885 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-27 07:24:13,886 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-27 07:24:13,886 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-27 07:24:13,887 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-27 07:24:13,888 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-27 07:24:13,890 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-27 07:24:13,891 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-27 07:24:13,891 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-27 07:24:13,893 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-27T07:24:14,140 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b939589-5539-0f0f-ceb6-48662b1efadc 2024-11-27 07:24:14,143 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-27 07:24:14,143 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-27T07:24:14,151 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-27T07:24:14,172 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-27T07:24:14,191 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b939589-5539-0f0f-ceb6-48662b1efadc/cluster_8a8651c6-cd4e-11be-2ecd-ca6a64beeda5, deleteOnExit=true 2024-11-27T07:24:14,191 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-27T07:24:14,193 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b939589-5539-0f0f-ceb6-48662b1efadc/test.cache.data in system properties and HBase conf 2024-11-27T07:24:14,194 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b939589-5539-0f0f-ceb6-48662b1efadc/hadoop.tmp.dir in system properties and HBase conf 2024-11-27T07:24:14,195 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b939589-5539-0f0f-ceb6-48662b1efadc/hadoop.log.dir in system properties and HBase conf 2024-11-27T07:24:14,195 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b939589-5539-0f0f-ceb6-48662b1efadc/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-27T07:24:14,196 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b939589-5539-0f0f-ceb6-48662b1efadc/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-27T07:24:14,197 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-27T07:24:14,287 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-27T07:24:14,374 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-27T07:24:14,378 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b939589-5539-0f0f-ceb6-48662b1efadc/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-27T07:24:14,378 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b939589-5539-0f0f-ceb6-48662b1efadc/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-27T07:24:14,379 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b939589-5539-0f0f-ceb6-48662b1efadc/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-27T07:24:14,379 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b939589-5539-0f0f-ceb6-48662b1efadc/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-27T07:24:14,379 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b939589-5539-0f0f-ceb6-48662b1efadc/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-27T07:24:14,380 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b939589-5539-0f0f-ceb6-48662b1efadc/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-27T07:24:14,380 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b939589-5539-0f0f-ceb6-48662b1efadc/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-27T07:24:14,381 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b939589-5539-0f0f-ceb6-48662b1efadc/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-27T07:24:14,381 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b939589-5539-0f0f-ceb6-48662b1efadc/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-27T07:24:14,381 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b939589-5539-0f0f-ceb6-48662b1efadc/nfs.dump.dir in system properties and HBase conf 2024-11-27T07:24:14,382 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b939589-5539-0f0f-ceb6-48662b1efadc/java.io.tmpdir in system properties and HBase conf 2024-11-27T07:24:14,382 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b939589-5539-0f0f-ceb6-48662b1efadc/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-27T07:24:14,382 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b939589-5539-0f0f-ceb6-48662b1efadc/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-27T07:24:14,383 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b939589-5539-0f0f-ceb6-48662b1efadc/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-27T07:24:15,493 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-27T07:24:15,571 INFO [Time-limited test {}] log.Log(170): Logging initialized @2563ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-27T07:24:15,638 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-27T07:24:15,691 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-27T07:24:15,709 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-27T07:24:15,709 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-27T07:24:15,710 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-27T07:24:15,721 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-27T07:24:15,724 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b939589-5539-0f0f-ceb6-48662b1efadc/hadoop.log.dir/,AVAILABLE} 2024-11-27T07:24:15,725 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-27T07:24:15,904 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b939589-5539-0f0f-ceb6-48662b1efadc/java.io.tmpdir/jetty-localhost-40299-hadoop-hdfs-3_4_1-tests_jar-_-any-3675253280759332660/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-27T07:24:15,910 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:40299} 2024-11-27T07:24:15,910 INFO [Time-limited test {}] server.Server(415): Started @2902ms 2024-11-27T07:24:16,703 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-27T07:24:16,710 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-27T07:24:16,711 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-27T07:24:16,712 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-27T07:24:16,712 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-27T07:24:16,712 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@336e9627{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b939589-5539-0f0f-ceb6-48662b1efadc/hadoop.log.dir/,AVAILABLE} 2024-11-27T07:24:16,713 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@42946cd0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-27T07:24:16,821 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2c68b1bf{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b939589-5539-0f0f-ceb6-48662b1efadc/java.io.tmpdir/jetty-localhost-43163-hadoop-hdfs-3_4_1-tests_jar-_-any-9406874163614118495/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-27T07:24:16,822 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3efe07f8{HTTP/1.1, (http/1.1)}{localhost:43163} 2024-11-27T07:24:16,822 INFO [Time-limited test {}] server.Server(415): Started @3815ms 2024-11-27T07:24:16,879 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-27T07:24:17,852 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b939589-5539-0f0f-ceb6-48662b1efadc/cluster_8a8651c6-cd4e-11be-2ecd-ca6a64beeda5/data/data2/current/BP-336076723-172.17.0.3-1732692254946/current, will proceed with Du for space computation calculation, 2024-11-27T07:24:17,852 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b939589-5539-0f0f-ceb6-48662b1efadc/cluster_8a8651c6-cd4e-11be-2ecd-ca6a64beeda5/data/data1/current/BP-336076723-172.17.0.3-1732692254946/current, will proceed with Du for space computation calculation, 2024-11-27T07:24:17,882 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-27T07:24:17,928 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3f8ec21417f4ba5e with lease ID 0x634fd6a87df6059e: Processing first storage report for DS-69978c75-fbd8-4f08-9252-b8cfd51b6e74 from datanode DatanodeRegistration(127.0.0.1:44229, datanodeUuid=96c860c6-dc59-478e-903a-a301ba3afd05, infoPort=43197, infoSecurePort=0, ipcPort=44445, storageInfo=lv=-57;cid=testClusterID;nsid=918779849;c=1732692254946) 2024-11-27T07:24:17,930 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3f8ec21417f4ba5e with lease ID 0x634fd6a87df6059e: from storage DS-69978c75-fbd8-4f08-9252-b8cfd51b6e74 node DatanodeRegistration(127.0.0.1:44229, datanodeUuid=96c860c6-dc59-478e-903a-a301ba3afd05, infoPort=43197, infoSecurePort=0, ipcPort=44445, storageInfo=lv=-57;cid=testClusterID;nsid=918779849;c=1732692254946), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-27T07:24:17,930 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3f8ec21417f4ba5e with lease ID 0x634fd6a87df6059e: Processing first storage report for DS-4bf44c87-0dc6-403d-ac7f-cded790f9719 from datanode DatanodeRegistration(127.0.0.1:44229, datanodeUuid=96c860c6-dc59-478e-903a-a301ba3afd05, infoPort=43197, infoSecurePort=0, ipcPort=44445, storageInfo=lv=-57;cid=testClusterID;nsid=918779849;c=1732692254946) 2024-11-27T07:24:17,930 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3f8ec21417f4ba5e with lease ID 0x634fd6a87df6059e: from storage DS-4bf44c87-0dc6-403d-ac7f-cded790f9719 node DatanodeRegistration(127.0.0.1:44229, datanodeUuid=96c860c6-dc59-478e-903a-a301ba3afd05, infoPort=43197, infoSecurePort=0, ipcPort=44445, storageInfo=lv=-57;cid=testClusterID;nsid=918779849;c=1732692254946), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-27T07:24:17,972 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b939589-5539-0f0f-ceb6-48662b1efadc 2024-11-27T07:24:18,044 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b939589-5539-0f0f-ceb6-48662b1efadc/cluster_8a8651c6-cd4e-11be-2ecd-ca6a64beeda5/zookeeper_0, clientPort=51145, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b939589-5539-0f0f-ceb6-48662b1efadc/cluster_8a8651c6-cd4e-11be-2ecd-ca6a64beeda5/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b939589-5539-0f0f-ceb6-48662b1efadc/cluster_8a8651c6-cd4e-11be-2ecd-ca6a64beeda5/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-27T07:24:18,055 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=51145 2024-11-27T07:24:18,073 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T07:24:18,077 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T07:24:18,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741825_1001 (size=7) 2024-11-27T07:24:18,757 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4 with version=8 2024-11-27T07:24:18,757 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/hbase-staging 2024-11-27T07:24:18,842 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-27T07:24:19,091 INFO [Time-limited test {}] client.ConnectionUtils(128): master/289954c9202d:0 server-side Connection retries=45 2024-11-27T07:24:19,100 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T07:24:19,100 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-27T07:24:19,104 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-27T07:24:19,104 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T07:24:19,104 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-27T07:24:19,224 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-27T07:24:19,273 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-27T07:24:19,281 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-27T07:24:19,284 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-27T07:24:19,306 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 32816 (auto-detected) 2024-11-27T07:24:19,306 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-27T07:24:19,322 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:43057 2024-11-27T07:24:19,341 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43057 connecting to ZooKeeper ensemble=127.0.0.1:51145 2024-11-27T07:24:19,469 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:430570x0, quorum=127.0.0.1:51145, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-27T07:24:19,472 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43057-0x1017b2da8960000 connected 2024-11-27T07:24:19,560 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T07:24:19,562 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T07:24:19,571 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43057-0x1017b2da8960000, quorum=127.0.0.1:51145, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T07:24:19,575 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4, hbase.cluster.distributed=false 2024-11-27T07:24:19,597 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43057-0x1017b2da8960000, quorum=127.0.0.1:51145, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-27T07:24:19,604 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43057 2024-11-27T07:24:19,605 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43057 2024-11-27T07:24:19,605 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43057 2024-11-27T07:24:19,606 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43057 2024-11-27T07:24:19,606 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43057 2024-11-27T07:24:19,703 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/289954c9202d:0 server-side Connection retries=45 2024-11-27T07:24:19,704 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T07:24:19,705 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-27T07:24:19,705 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-27T07:24:19,705 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T07:24:19,705 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-27T07:24:19,708 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-27T07:24:19,710 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-27T07:24:19,711 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:42117 2024-11-27T07:24:19,713 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42117 connecting to ZooKeeper ensemble=127.0.0.1:51145 2024-11-27T07:24:19,715 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T07:24:19,717 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T07:24:19,734 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:421170x0, quorum=127.0.0.1:51145, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-27T07:24:19,735 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42117-0x1017b2da8960001 connected 2024-11-27T07:24:19,735 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42117-0x1017b2da8960001, quorum=127.0.0.1:51145, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T07:24:19,740 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-27T07:24:19,750 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-27T07:24:19,752 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42117-0x1017b2da8960001, quorum=127.0.0.1:51145, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-27T07:24:19,757 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42117-0x1017b2da8960001, quorum=127.0.0.1:51145, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-27T07:24:19,758 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42117 2024-11-27T07:24:19,758 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42117 2024-11-27T07:24:19,759 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42117 2024-11-27T07:24:19,759 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42117 2024-11-27T07:24:19,760 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42117 2024-11-27T07:24:19,775 DEBUG [M:0;289954c9202d:43057 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;289954c9202d:43057 2024-11-27T07:24:19,775 INFO [master/289954c9202d:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/289954c9202d,43057,1732692258941 2024-11-27T07:24:19,791 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43057-0x1017b2da8960000, quorum=127.0.0.1:51145, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T07:24:19,791 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42117-0x1017b2da8960001, quorum=127.0.0.1:51145, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T07:24:19,793 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43057-0x1017b2da8960000, quorum=127.0.0.1:51145, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/289954c9202d,43057,1732692258941 2024-11-27T07:24:19,822 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42117-0x1017b2da8960001, quorum=127.0.0.1:51145, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-27T07:24:19,822 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43057-0x1017b2da8960000, quorum=127.0.0.1:51145, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T07:24:19,822 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42117-0x1017b2da8960001, quorum=127.0.0.1:51145, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T07:24:19,823 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43057-0x1017b2da8960000, quorum=127.0.0.1:51145, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-27T07:24:19,824 INFO [master/289954c9202d:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/289954c9202d,43057,1732692258941 from backup master directory 2024-11-27T07:24:19,832 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43057-0x1017b2da8960000, quorum=127.0.0.1:51145, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/289954c9202d,43057,1732692258941 2024-11-27T07:24:19,832 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42117-0x1017b2da8960001, quorum=127.0.0.1:51145, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T07:24:19,833 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43057-0x1017b2da8960000, quorum=127.0.0.1:51145, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T07:24:19,833 WARN [master/289954c9202d:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-27T07:24:19,833 INFO [master/289954c9202d:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=289954c9202d,43057,1732692258941 2024-11-27T07:24:19,835 INFO [master/289954c9202d:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-27T07:24:19,836 INFO [master/289954c9202d:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-27T07:24:19,886 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/hbase.id] with ID: 09f43d3f-c175-4713-a9ce-038f5695b74b 2024-11-27T07:24:19,887 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/.tmp/hbase.id 2024-11-27T07:24:19,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741826_1002 (size=42) 2024-11-27T07:24:19,897 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/.tmp/hbase.id]:[hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/hbase.id] 2024-11-27T07:24:19,947 INFO [master/289954c9202d:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T07:24:19,953 INFO [master/289954c9202d:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-27T07:24:19,972 INFO [master/289954c9202d:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms. 2024-11-27T07:24:20,034 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43057-0x1017b2da8960000, quorum=127.0.0.1:51145, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T07:24:20,034 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42117-0x1017b2da8960001, quorum=127.0.0.1:51145, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T07:24:20,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741827_1003 (size=196) 2024-11-27T07:24:20,069 INFO [master/289954c9202d:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T07:24:20,071 INFO [master/289954c9202d:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-27T07:24:20,086 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:20,090 INFO [master/289954c9202d:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T07:24:20,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741828_1004 (size=1189) 2024-11-27T07:24:20,542 INFO [master/289954c9202d:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/MasterData/data/master/store 2024-11-27T07:24:20,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741829_1005 (size=34) 2024-11-27T07:24:20,970 INFO [master/289954c9202d:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-27T07:24:20,974 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T07:24:20,974 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-27T07:24:20,975 INFO [master/289954c9202d:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T07:24:20,975 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T07:24:20,976 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-27T07:24:20,976 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T07:24:20,976 INFO [master/289954c9202d:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T07:24:20,978 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732692260974Disabling compacts and flushes for region at 1732692260974Disabling writes for close at 1732692260976 (+2 ms)Writing region close event to WAL at 1732692260976Closed at 1732692260976 2024-11-27T07:24:20,980 WARN [master/289954c9202d:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/MasterData/data/master/store/.initializing 2024-11-27T07:24:20,980 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/MasterData/WALs/289954c9202d,43057,1732692258941 2024-11-27T07:24:20,989 INFO [master/289954c9202d:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T07:24:21,004 INFO [master/289954c9202d:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=289954c9202d%2C43057%2C1732692258941, suffix=, logDir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/MasterData/WALs/289954c9202d,43057,1732692258941, archiveDir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/MasterData/oldWALs, maxLogs=10 2024-11-27T07:24:21,025 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/MasterData/WALs/289954c9202d,43057,1732692258941/289954c9202d%2C43057%2C1732692258941.1732692261008, exclude list is [], retry=0 2024-11-27T07:24:21,040 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44229,DS-69978c75-fbd8-4f08-9252-b8cfd51b6e74,DISK] 2024-11-27T07:24:21,043 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-27T07:24:21,075 INFO [master/289954c9202d:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/MasterData/WALs/289954c9202d,43057,1732692258941/289954c9202d%2C43057%2C1732692258941.1732692261008 2024-11-27T07:24:21,076 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43197:43197)] 2024-11-27T07:24:21,076 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-27T07:24:21,077 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T07:24:21,080 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T07:24:21,081 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T07:24:21,115 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T07:24:21,139 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-27T07:24:21,142 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:21,145 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T07:24:21,145 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T07:24:21,148 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-27T07:24:21,149 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:21,150 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T07:24:21,150 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T07:24:21,153 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-27T07:24:21,153 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:21,154 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T07:24:21,154 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T07:24:21,157 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-27T07:24:21,157 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:21,158 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T07:24:21,158 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T07:24:21,163 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-27T07:24:21,164 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-27T07:24:21,170 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T07:24:21,171 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T07:24:21,175 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-27T07:24:21,179 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T07:24:21,183 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T07:24:21,185 INFO [master/289954c9202d:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59524492, jitterRate=-0.11301594972610474}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-27T07:24:21,192 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732692261092Initializing all the Stores at 1732692261094 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732692261095 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732692261095Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732692261096 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732692261096Cleaning up temporary data from old regions at 1732692261171 (+75 ms)Region opened successfully at 1732692261192 (+21 ms) 2024-11-27T07:24:21,193 INFO [master/289954c9202d:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-27T07:24:21,227 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f654f69, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=289954c9202d/172.17.0.3:0 2024-11-27T07:24:21,256 INFO [master/289954c9202d:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-27T07:24:21,266 INFO [master/289954c9202d:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-27T07:24:21,266 INFO [master/289954c9202d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-27T07:24:21,268 INFO [master/289954c9202d:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-27T07:24:21,270 INFO [master/289954c9202d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-27T07:24:21,275 INFO [master/289954c9202d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-27T07:24:21,276 INFO [master/289954c9202d:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-27T07:24:21,302 INFO [master/289954c9202d:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-27T07:24:21,312 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43057-0x1017b2da8960000, quorum=127.0.0.1:51145, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-27T07:24:21,377 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-27T07:24:21,382 INFO [master/289954c9202d:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-27T07:24:21,386 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43057-0x1017b2da8960000, quorum=127.0.0.1:51145, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-27T07:24:21,397 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-27T07:24:21,399 INFO [master/289954c9202d:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-27T07:24:21,403 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43057-0x1017b2da8960000, quorum=127.0.0.1:51145, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-27T07:24:21,411 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-27T07:24:21,413 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43057-0x1017b2da8960000, quorum=127.0.0.1:51145, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-27T07:24:21,421 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-27T07:24:21,443 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43057-0x1017b2da8960000, quorum=127.0.0.1:51145, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-27T07:24:21,453 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-27T07:24:21,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43057-0x1017b2da8960000, quorum=127.0.0.1:51145, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-27T07:24:21,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42117-0x1017b2da8960001, quorum=127.0.0.1:51145, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-27T07:24:21,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43057-0x1017b2da8960000, quorum=127.0.0.1:51145, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T07:24:21,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42117-0x1017b2da8960001, quorum=127.0.0.1:51145, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T07:24:21,467 INFO [master/289954c9202d:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=289954c9202d,43057,1732692258941, sessionid=0x1017b2da8960000, setting cluster-up flag (Was=false) 2024-11-27T07:24:21,575 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43057-0x1017b2da8960000, quorum=127.0.0.1:51145, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T07:24:21,575 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42117-0x1017b2da8960001, quorum=127.0.0.1:51145, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T07:24:21,622 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-27T07:24:21,627 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=289954c9202d,43057,1732692258941 2024-11-27T07:24:21,653 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43057-0x1017b2da8960000, quorum=127.0.0.1:51145, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T07:24:21,653 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42117-0x1017b2da8960001, quorum=127.0.0.1:51145, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T07:24:21,685 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-27T07:24:21,688 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=289954c9202d,43057,1732692258941 2024-11-27T07:24:21,697 INFO [master/289954c9202d:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-27T07:24:21,763 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-27T07:24:21,765 INFO [RS:0;289954c9202d:42117 {}] regionserver.HRegionServer(746): ClusterId : 09f43d3f-c175-4713-a9ce-038f5695b74b 2024-11-27T07:24:21,767 DEBUG [RS:0;289954c9202d:42117 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-27T07:24:21,772 INFO [master/289954c9202d:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-27T07:24:21,778 INFO [master/289954c9202d:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-27T07:24:21,781 DEBUG [RS:0;289954c9202d:42117 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-27T07:24:21,782 DEBUG [RS:0;289954c9202d:42117 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-27T07:24:21,784 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 289954c9202d,43057,1732692258941 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-27T07:24:21,791 DEBUG [RS:0;289954c9202d:42117 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-27T07:24:21,792 DEBUG [RS:0;289954c9202d:42117 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d9c6584, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=289954c9202d/172.17.0.3:0 2024-11-27T07:24:21,792 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/289954c9202d:0, corePoolSize=5, maxPoolSize=5 2024-11-27T07:24:21,793 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/289954c9202d:0, corePoolSize=5, maxPoolSize=5 2024-11-27T07:24:21,793 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/289954c9202d:0, corePoolSize=5, maxPoolSize=5 2024-11-27T07:24:21,793 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/289954c9202d:0, corePoolSize=5, maxPoolSize=5 2024-11-27T07:24:21,793 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/289954c9202d:0, corePoolSize=10, maxPoolSize=10 2024-11-27T07:24:21,793 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/289954c9202d:0, corePoolSize=1, maxPoolSize=1 2024-11-27T07:24:21,793 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/289954c9202d:0, corePoolSize=2, maxPoolSize=2 2024-11-27T07:24:21,793 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/289954c9202d:0, corePoolSize=1, maxPoolSize=1 2024-11-27T07:24:21,795 INFO [master/289954c9202d:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732692291794 2024-11-27T07:24:21,796 INFO [master/289954c9202d:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-27T07:24:21,797 INFO [master/289954c9202d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-27T07:24:21,798 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-27T07:24:21,799 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-27T07:24:21,800 INFO [master/289954c9202d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-27T07:24:21,800 INFO [master/289954c9202d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-27T07:24:21,801 INFO [master/289954c9202d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-27T07:24:21,801 INFO [master/289954c9202d:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-27T07:24:21,804 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:21,804 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-27T07:24:21,804 INFO [master/289954c9202d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-27T07:24:21,807 INFO [master/289954c9202d:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-27T07:24:21,808 INFO [master/289954c9202d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-27T07:24:21,809 INFO [master/289954c9202d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-27T07:24:21,810 DEBUG [RS:0;289954c9202d:42117 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;289954c9202d:42117 2024-11-27T07:24:21,811 INFO [master/289954c9202d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-27T07:24:21,812 INFO [master/289954c9202d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-27T07:24:21,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741831_1007 (size=1321) 2024-11-27T07:24:21,814 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/289954c9202d:0:becomeActiveMaster-HFileCleaner.large.0-1732692261813,5,FailOnTimeoutGroup] 2024-11-27T07:24:21,814 INFO [RS:0;289954c9202d:42117 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-27T07:24:21,815 INFO [RS:0;289954c9202d:42117 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-27T07:24:21,815 DEBUG [RS:0;289954c9202d:42117 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-27T07:24:21,815 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/289954c9202d:0:becomeActiveMaster-HFileCleaner.small.0-1732692261814,5,FailOnTimeoutGroup] 2024-11-27T07:24:21,815 INFO [master/289954c9202d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-27T07:24:21,815 INFO [master/289954c9202d:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-27T07:24:21,817 INFO [master/289954c9202d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-27T07:24:21,817 INFO [master/289954c9202d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-27T07:24:21,817 INFO [RS:0;289954c9202d:42117 {}] regionserver.HRegionServer(2659): reportForDuty to master=289954c9202d,43057,1732692258941 with port=42117, startcode=1732692259671 2024-11-27T07:24:21,826 DEBUG [RS:0;289954c9202d:42117 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-27T07:24:21,858 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45941, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-27T07:24:21,864 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43057 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 289954c9202d,42117,1732692259671 2024-11-27T07:24:21,866 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43057 {}] master.ServerManager(517): Registering regionserver=289954c9202d,42117,1732692259671 2024-11-27T07:24:21,880 DEBUG [RS:0;289954c9202d:42117 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4 2024-11-27T07:24:21,881 DEBUG [RS:0;289954c9202d:42117 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:39369 2024-11-27T07:24:21,881 DEBUG [RS:0;289954c9202d:42117 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-27T07:24:21,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43057-0x1017b2da8960000, quorum=127.0.0.1:51145, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-27T07:24:21,896 DEBUG [RS:0;289954c9202d:42117 {}] zookeeper.ZKUtil(111): regionserver:42117-0x1017b2da8960001, quorum=127.0.0.1:51145, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/289954c9202d,42117,1732692259671 2024-11-27T07:24:21,896 WARN [RS:0;289954c9202d:42117 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-27T07:24:21,897 INFO [RS:0;289954c9202d:42117 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T07:24:21,897 DEBUG [RS:0;289954c9202d:42117 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/WALs/289954c9202d,42117,1732692259671 2024-11-27T07:24:21,899 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [289954c9202d,42117,1732692259671] 2024-11-27T07:24:21,922 INFO [RS:0;289954c9202d:42117 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-27T07:24:21,937 INFO [RS:0;289954c9202d:42117 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-27T07:24:21,942 INFO [RS:0;289954c9202d:42117 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-27T07:24:21,942 INFO [RS:0;289954c9202d:42117 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T07:24:21,943 INFO [RS:0;289954c9202d:42117 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-27T07:24:21,947 INFO [RS:0;289954c9202d:42117 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-27T07:24:21,948 INFO [RS:0;289954c9202d:42117 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-27T07:24:21,949 DEBUG [RS:0;289954c9202d:42117 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/289954c9202d:0, corePoolSize=1, maxPoolSize=1 2024-11-27T07:24:21,949 DEBUG [RS:0;289954c9202d:42117 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/289954c9202d:0, corePoolSize=1, maxPoolSize=1 2024-11-27T07:24:21,949 DEBUG [RS:0;289954c9202d:42117 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/289954c9202d:0, corePoolSize=1, maxPoolSize=1 2024-11-27T07:24:21,949 DEBUG [RS:0;289954c9202d:42117 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/289954c9202d:0, corePoolSize=1, maxPoolSize=1 2024-11-27T07:24:21,949 DEBUG [RS:0;289954c9202d:42117 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/289954c9202d:0, corePoolSize=1, maxPoolSize=1 2024-11-27T07:24:21,949 DEBUG [RS:0;289954c9202d:42117 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/289954c9202d:0, corePoolSize=2, maxPoolSize=2 2024-11-27T07:24:21,949 DEBUG [RS:0;289954c9202d:42117 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/289954c9202d:0, corePoolSize=1, maxPoolSize=1 2024-11-27T07:24:21,950 DEBUG [RS:0;289954c9202d:42117 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/289954c9202d:0, corePoolSize=1, maxPoolSize=1 2024-11-27T07:24:21,950 DEBUG [RS:0;289954c9202d:42117 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/289954c9202d:0, corePoolSize=1, maxPoolSize=1 2024-11-27T07:24:21,950 DEBUG [RS:0;289954c9202d:42117 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/289954c9202d:0, corePoolSize=1, maxPoolSize=1 2024-11-27T07:24:21,950 DEBUG [RS:0;289954c9202d:42117 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/289954c9202d:0, corePoolSize=1, maxPoolSize=1 2024-11-27T07:24:21,950 DEBUG [RS:0;289954c9202d:42117 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/289954c9202d:0, corePoolSize=1, maxPoolSize=1 2024-11-27T07:24:21,950 DEBUG [RS:0;289954c9202d:42117 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/289954c9202d:0, corePoolSize=3, maxPoolSize=3 2024-11-27T07:24:21,951 DEBUG [RS:0;289954c9202d:42117 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0, corePoolSize=3, maxPoolSize=3 2024-11-27T07:24:21,951 INFO [RS:0;289954c9202d:42117 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T07:24:21,952 INFO [RS:0;289954c9202d:42117 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T07:24:21,952 INFO [RS:0;289954c9202d:42117 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T07:24:21,952 INFO [RS:0;289954c9202d:42117 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-27T07:24:21,952 INFO [RS:0;289954c9202d:42117 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-27T07:24:21,952 INFO [RS:0;289954c9202d:42117 {}] hbase.ChoreService(168): Chore ScheduledChore name=289954c9202d,42117,1732692259671-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-27T07:24:21,970 INFO [RS:0;289954c9202d:42117 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-27T07:24:21,972 INFO [RS:0;289954c9202d:42117 {}] hbase.ChoreService(168): Chore ScheduledChore name=289954c9202d,42117,1732692259671-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T07:24:21,972 INFO [RS:0;289954c9202d:42117 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T07:24:21,972 INFO [RS:0;289954c9202d:42117 {}] regionserver.Replication(171): 289954c9202d,42117,1732692259671 started 2024-11-27T07:24:21,990 INFO [RS:0;289954c9202d:42117 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T07:24:21,990 INFO [RS:0;289954c9202d:42117 {}] regionserver.HRegionServer(1482): Serving as 289954c9202d,42117,1732692259671, RpcServer on 289954c9202d/172.17.0.3:42117, sessionid=0x1017b2da8960001 2024-11-27T07:24:21,991 DEBUG [RS:0;289954c9202d:42117 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-27T07:24:21,991 DEBUG [RS:0;289954c9202d:42117 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 289954c9202d,42117,1732692259671 2024-11-27T07:24:21,991 DEBUG [RS:0;289954c9202d:42117 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '289954c9202d,42117,1732692259671' 2024-11-27T07:24:21,992 DEBUG [RS:0;289954c9202d:42117 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-27T07:24:21,993 DEBUG [RS:0;289954c9202d:42117 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-27T07:24:21,994 DEBUG [RS:0;289954c9202d:42117 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-27T07:24:21,994 DEBUG [RS:0;289954c9202d:42117 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-27T07:24:21,994 DEBUG [RS:0;289954c9202d:42117 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 289954c9202d,42117,1732692259671 2024-11-27T07:24:21,994 DEBUG [RS:0;289954c9202d:42117 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '289954c9202d,42117,1732692259671' 2024-11-27T07:24:21,994 DEBUG [RS:0;289954c9202d:42117 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-27T07:24:21,995 DEBUG [RS:0;289954c9202d:42117 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-27T07:24:21,996 DEBUG [RS:0;289954c9202d:42117 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-27T07:24:21,996 INFO [RS:0;289954c9202d:42117 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-27T07:24:21,996 INFO [RS:0;289954c9202d:42117 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-27T07:24:22,105 INFO [RS:0;289954c9202d:42117 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T07:24:22,109 INFO [RS:0;289954c9202d:42117 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=289954c9202d%2C42117%2C1732692259671, suffix=, logDir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/WALs/289954c9202d,42117,1732692259671, archiveDir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/oldWALs, maxLogs=32 2024-11-27T07:24:22,126 DEBUG [RS:0;289954c9202d:42117 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/WALs/289954c9202d,42117,1732692259671/289954c9202d%2C42117%2C1732692259671.1732692262112, exclude list is [], retry=0 2024-11-27T07:24:22,130 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44229,DS-69978c75-fbd8-4f08-9252-b8cfd51b6e74,DISK] 2024-11-27T07:24:22,133 INFO [RS:0;289954c9202d:42117 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/WALs/289954c9202d,42117,1732692259671/289954c9202d%2C42117%2C1732692259671.1732692262112 2024-11-27T07:24:22,134 DEBUG [RS:0;289954c9202d:42117 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43197:43197)] 2024-11-27T07:24:22,217 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-27T07:24:22,218 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4 2024-11-27T07:24:22,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741833_1009 (size=32) 2024-11-27T07:24:22,636 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T07:24:22,641 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-27T07:24:22,645 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-27T07:24:22,645 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:22,646 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T07:24:22,646 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-27T07:24:22,649 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-27T07:24:22,649 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:22,650 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T07:24:22,650 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-27T07:24:22,653 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-27T07:24:22,653 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:22,654 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T07:24:22,654 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-27T07:24:22,657 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-27T07:24:22,657 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:22,658 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T07:24:22,658 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-27T07:24:22,659 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/hbase/meta/1588230740 2024-11-27T07:24:22,660 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/hbase/meta/1588230740 2024-11-27T07:24:22,662 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-27T07:24:22,662 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-27T07:24:22,663 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T07:24:22,665 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-27T07:24:22,669 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T07:24:22,670 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66936968, jitterRate=-0.0025614500045776367}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T07:24:22,672 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732692262637Initializing all the Stores at 1732692262641 (+4 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732692262641Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732692262641Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732692262641Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732692262641Cleaning up temporary data from old regions at 1732692262663 (+22 ms)Region opened successfully at 1732692262672 (+9 ms) 2024-11-27T07:24:22,673 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-27T07:24:22,673 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-27T07:24:22,673 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-27T07:24:22,673 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-27T07:24:22,673 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-27T07:24:22,674 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-27T07:24:22,674 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732692262673Disabling compacts and flushes for region at 1732692262673Disabling writes for close at 1732692262673Writing region close event to WAL at 1732692262674 (+1 ms)Closed at 1732692262674 2024-11-27T07:24:22,678 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-27T07:24:22,678 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-27T07:24:22,684 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-27T07:24:22,691 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T07:24:22,694 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-27T07:24:22,851 DEBUG [289954c9202d:43057 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-27T07:24:22,862 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=289954c9202d,42117,1732692259671 2024-11-27T07:24:22,869 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 289954c9202d,42117,1732692259671, state=OPENING 2024-11-27T07:24:22,913 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-27T07:24:22,963 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42117-0x1017b2da8960001, quorum=127.0.0.1:51145, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T07:24:22,963 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43057-0x1017b2da8960000, quorum=127.0.0.1:51145, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T07:24:22,964 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T07:24:22,964 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T07:24:22,965 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T07:24:22,967 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=289954c9202d,42117,1732692259671}] 2024-11-27T07:24:23,145 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-27T07:24:23,148 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58537, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-27T07:24:23,160 INFO [RS_OPEN_META-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-27T07:24:23,161 INFO [RS_OPEN_META-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T07:24:23,162 INFO [RS_OPEN_META-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-27T07:24:23,165 INFO [RS_OPEN_META-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=289954c9202d%2C42117%2C1732692259671.meta, suffix=.meta, logDir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/WALs/289954c9202d,42117,1732692259671, archiveDir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/oldWALs, maxLogs=32 2024-11-27T07:24:23,180 DEBUG [RS_OPEN_META-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/WALs/289954c9202d,42117,1732692259671/289954c9202d%2C42117%2C1732692259671.meta.1732692263167.meta, exclude list is [], retry=0 2024-11-27T07:24:23,184 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44229,DS-69978c75-fbd8-4f08-9252-b8cfd51b6e74,DISK] 2024-11-27T07:24:23,187 INFO [RS_OPEN_META-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/WALs/289954c9202d,42117,1732692259671/289954c9202d%2C42117%2C1732692259671.meta.1732692263167.meta 2024-11-27T07:24:23,187 DEBUG [RS_OPEN_META-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43197:43197)] 2024-11-27T07:24:23,188 DEBUG [RS_OPEN_META-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-27T07:24:23,190 DEBUG [RS_OPEN_META-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-27T07:24:23,193 DEBUG [RS_OPEN_META-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-27T07:24:23,197 INFO [RS_OPEN_META-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-27T07:24:23,201 DEBUG [RS_OPEN_META-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-27T07:24:23,202 DEBUG [RS_OPEN_META-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T07:24:23,203 DEBUG [RS_OPEN_META-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-27T07:24:23,203 DEBUG [RS_OPEN_META-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-27T07:24:23,206 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-27T07:24:23,207 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-27T07:24:23,207 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:23,208 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T07:24:23,208 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-27T07:24:23,210 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-27T07:24:23,210 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:23,210 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T07:24:23,211 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-27T07:24:23,212 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-27T07:24:23,212 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:23,213 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T07:24:23,213 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-27T07:24:23,214 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-27T07:24:23,214 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:23,215 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T07:24:23,215 DEBUG [RS_OPEN_META-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-27T07:24:23,217 DEBUG [RS_OPEN_META-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/hbase/meta/1588230740 2024-11-27T07:24:23,219 DEBUG [RS_OPEN_META-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/hbase/meta/1588230740 2024-11-27T07:24:23,222 DEBUG [RS_OPEN_META-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-27T07:24:23,222 DEBUG [RS_OPEN_META-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-27T07:24:23,223 DEBUG [RS_OPEN_META-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T07:24:23,226 DEBUG [RS_OPEN_META-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-27T07:24:23,229 INFO [RS_OPEN_META-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70102522, jitterRate=0.04460898041725159}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T07:24:23,229 DEBUG [RS_OPEN_META-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-27T07:24:23,232 DEBUG [RS_OPEN_META-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732692263203Writing region info on filesystem at 1732692263204 (+1 ms)Initializing all the Stores at 1732692263205 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732692263206 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732692263206Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732692263206Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732692263206Cleaning up temporary data from old regions at 1732692263222 (+16 ms)Running coprocessor post-open hooks at 1732692263229 (+7 ms)Region opened successfully at 1732692263232 (+3 ms) 2024-11-27T07:24:23,239 INFO [RS_OPEN_META-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732692263134 2024-11-27T07:24:23,249 DEBUG [RS_OPEN_META-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-27T07:24:23,250 INFO [RS_OPEN_META-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-27T07:24:23,252 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=289954c9202d,42117,1732692259671 2024-11-27T07:24:23,254 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 289954c9202d,42117,1732692259671, state=OPEN 2024-11-27T07:24:23,310 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43057-0x1017b2da8960000, quorum=127.0.0.1:51145, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T07:24:23,310 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42117-0x1017b2da8960001, quorum=127.0.0.1:51145, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T07:24:23,310 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T07:24:23,310 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T07:24:23,310 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:23,319 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-27T07:24:23,320 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=289954c9202d,42117,1732692259671 in 344 msec 2024-11-27T07:24:23,326 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-27T07:24:23,326 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 638 msec 2024-11-27T07:24:23,327 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-27T07:24:23,327 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-27T07:24:23,344 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T07:24:23,345 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=289954c9202d,42117,1732692259671, seqNum=-1] 2024-11-27T07:24:23,362 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T07:24:23,364 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57783, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T07:24:23,385 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.6570 sec 2024-11-27T07:24:23,385 INFO [master/289954c9202d:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732692263385, completionTime=-1 2024-11-27T07:24:23,388 INFO [master/289954c9202d:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-27T07:24:23,388 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-27T07:24:23,434 INFO [master/289954c9202d:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-27T07:24:23,434 INFO [master/289954c9202d:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732692323434 2024-11-27T07:24:23,434 INFO [master/289954c9202d:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732692383434 2024-11-27T07:24:23,434 INFO [master/289954c9202d:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 46 msec 2024-11-27T07:24:23,436 INFO [master/289954c9202d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=289954c9202d,43057,1732692258941-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T07:24:23,436 INFO [master/289954c9202d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=289954c9202d,43057,1732692258941-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T07:24:23,436 INFO [master/289954c9202d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=289954c9202d,43057,1732692258941-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T07:24:23,441 INFO [master/289954c9202d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-289954c9202d:43057, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T07:24:23,441 INFO [master/289954c9202d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-27T07:24:23,442 INFO [master/289954c9202d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-27T07:24:23,447 DEBUG [master/289954c9202d:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-27T07:24:23,465 INFO [master/289954c9202d:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.632sec 2024-11-27T07:24:23,466 INFO [master/289954c9202d:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-27T07:24:23,467 INFO [master/289954c9202d:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-27T07:24:23,468 INFO [master/289954c9202d:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-27T07:24:23,468 INFO [master/289954c9202d:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-27T07:24:23,468 INFO [master/289954c9202d:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-27T07:24:23,469 INFO [master/289954c9202d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=289954c9202d,43057,1732692258941-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-27T07:24:23,469 INFO [master/289954c9202d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=289954c9202d,43057,1732692258941-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-27T07:24:23,476 DEBUG [master/289954c9202d:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-27T07:24:23,477 INFO [master/289954c9202d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-27T07:24:23,477 INFO [master/289954c9202d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=289954c9202d,43057,1732692258941-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T07:24:23,583 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67d2c9bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:24:23,584 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-27T07:24:23,584 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-27T07:24:23,587 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 289954c9202d,43057,-1 for getting cluster id 2024-11-27T07:24:23,589 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T07:24:23,595 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '09f43d3f-c175-4713-a9ce-038f5695b74b' 2024-11-27T07:24:23,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T07:24:23,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "09f43d3f-c175-4713-a9ce-038f5695b74b" 2024-11-27T07:24:23,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5abd05e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:24:23,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [289954c9202d,43057,-1] 2024-11-27T07:24:23,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T07:24:23,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:24:23,603 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52722, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T07:24:23,605 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a26ad16, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:24:23,605 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T07:24:23,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=289954c9202d,42117,1732692259671, seqNum=-1] 2024-11-27T07:24:23,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T07:24:23,615 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50028, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T07:24:23,633 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=289954c9202d,43057,1732692258941 2024-11-27T07:24:23,646 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=414, ProcessCount=11, AvailableMemoryMB=5822 2024-11-27T07:24:23,653 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-27T07:24:23,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 289954c9202d,43057,1732692258941 2024-11-27T07:24:23,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@219792a1 2024-11-27T07:24:23,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-27T07:24:23,663 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52728, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-27T07:24:23,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-27T07:24:23,669 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T07:24:23,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-27T07:24:23,679 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T07:24:23,681 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-27T07:24:23,682 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:23,684 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T07:24:23,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T07:24:23,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741835_1011 (size=959) 2024-11-27T07:24:23,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T07:24:24,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T07:24:24,104 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4 2024-11-27T07:24:24,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741836_1012 (size=53) 2024-11-27T07:24:24,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T07:24:24,516 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T07:24:24,516 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 18c429ab939f89ba8ff6429b6bcb0851, disabling compactions & flushes 2024-11-27T07:24:24,516 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:24,516 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:24,516 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. after waiting 0 ms 2024-11-27T07:24:24,517 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:24,517 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:24,517 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 18c429ab939f89ba8ff6429b6bcb0851: Waiting for close lock at 1732692264516Disabling compacts and flushes for region at 1732692264516Disabling writes for close at 1732692264517 (+1 ms)Writing region close event to WAL at 1732692264517Closed at 1732692264517 2024-11-27T07:24:24,521 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T07:24:24,527 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732692264521"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732692264521"}]},"ts":"1732692264521"} 2024-11-27T07:24:24,533 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T07:24:24,535 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T07:24:24,537 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732692264535"}]},"ts":"1732692264535"} 2024-11-27T07:24:24,541 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-27T07:24:24,543 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=18c429ab939f89ba8ff6429b6bcb0851, ASSIGN}] 2024-11-27T07:24:24,546 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=18c429ab939f89ba8ff6429b6bcb0851, ASSIGN 2024-11-27T07:24:24,548 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=18c429ab939f89ba8ff6429b6bcb0851, ASSIGN; state=OFFLINE, location=289954c9202d,42117,1732692259671; forceNewPlan=false, retain=false 2024-11-27T07:24:24,700 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=18c429ab939f89ba8ff6429b6bcb0851, regionState=OPENING, regionLocation=289954c9202d,42117,1732692259671 2024-11-27T07:24:24,705 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=18c429ab939f89ba8ff6429b6bcb0851, ASSIGN because future has completed 2024-11-27T07:24:24,706 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671}] 2024-11-27T07:24:24,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T07:24:24,869 INFO [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:24,869 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} 2024-11-27T07:24:24,870 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 18c429ab939f89ba8ff6429b6bcb0851 2024-11-27T07:24:24,870 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T07:24:24,870 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 18c429ab939f89ba8ff6429b6bcb0851 2024-11-27T07:24:24,870 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 18c429ab939f89ba8ff6429b6bcb0851 2024-11-27T07:24:24,873 INFO [StoreOpener-18c429ab939f89ba8ff6429b6bcb0851-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 18c429ab939f89ba8ff6429b6bcb0851 2024-11-27T07:24:24,875 INFO [StoreOpener-18c429ab939f89ba8ff6429b6bcb0851-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 18c429ab939f89ba8ff6429b6bcb0851 columnFamilyName A 2024-11-27T07:24:24,876 DEBUG [StoreOpener-18c429ab939f89ba8ff6429b6bcb0851-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:24,877 INFO [StoreOpener-18c429ab939f89ba8ff6429b6bcb0851-1 {}] regionserver.HStore(327): Store=18c429ab939f89ba8ff6429b6bcb0851/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T07:24:24,877 INFO [StoreOpener-18c429ab939f89ba8ff6429b6bcb0851-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 18c429ab939f89ba8ff6429b6bcb0851 2024-11-27T07:24:24,880 INFO [StoreOpener-18c429ab939f89ba8ff6429b6bcb0851-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 18c429ab939f89ba8ff6429b6bcb0851 columnFamilyName B 2024-11-27T07:24:24,880 DEBUG [StoreOpener-18c429ab939f89ba8ff6429b6bcb0851-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:24,882 INFO [StoreOpener-18c429ab939f89ba8ff6429b6bcb0851-1 {}] regionserver.HStore(327): Store=18c429ab939f89ba8ff6429b6bcb0851/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T07:24:24,882 INFO [StoreOpener-18c429ab939f89ba8ff6429b6bcb0851-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 18c429ab939f89ba8ff6429b6bcb0851 2024-11-27T07:24:24,885 INFO [StoreOpener-18c429ab939f89ba8ff6429b6bcb0851-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 18c429ab939f89ba8ff6429b6bcb0851 columnFamilyName C 2024-11-27T07:24:24,885 DEBUG [StoreOpener-18c429ab939f89ba8ff6429b6bcb0851-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:24,886 INFO [StoreOpener-18c429ab939f89ba8ff6429b6bcb0851-1 {}] regionserver.HStore(327): Store=18c429ab939f89ba8ff6429b6bcb0851/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T07:24:24,887 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 18c429ab939f89ba8ff6429b6bcb0851 2024-11-27T07:24:24,889 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851 2024-11-27T07:24:24,890 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851 2024-11-27T07:24:24,892 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 18c429ab939f89ba8ff6429b6bcb0851 2024-11-27T07:24:24,892 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 18c429ab939f89ba8ff6429b6bcb0851 2024-11-27T07:24:24,893 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T07:24:24,895 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 18c429ab939f89ba8ff6429b6bcb0851 2024-11-27T07:24:24,899 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T07:24:24,900 INFO [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 18c429ab939f89ba8ff6429b6bcb0851; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61955029, jitterRate=-0.07679812610149384}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T07:24:24,900 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 18c429ab939f89ba8ff6429b6bcb0851 2024-11-27T07:24:24,901 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 18c429ab939f89ba8ff6429b6bcb0851: Running coprocessor pre-open hook at 1732692264870Writing region info on filesystem at 1732692264870Initializing all the Stores at 1732692264872 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732692264872Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732692264873 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732692264873Cleaning up temporary data from old regions at 1732692264892 (+19 ms)Running coprocessor post-open hooks at 1732692264900 (+8 ms)Region opened successfully at 1732692264901 (+1 ms) 2024-11-27T07:24:24,904 INFO [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., pid=6, masterSystemTime=1732692264862 2024-11-27T07:24:24,908 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:24,909 INFO [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:24,910 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=18c429ab939f89ba8ff6429b6bcb0851, regionState=OPEN, openSeqNum=2, regionLocation=289954c9202d,42117,1732692259671 2024-11-27T07:24:24,915 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 because future has completed 2024-11-27T07:24:24,924 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-27T07:24:24,924 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 in 211 msec 2024-11-27T07:24:24,931 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-27T07:24:24,931 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=18c429ab939f89ba8ff6429b6bcb0851, ASSIGN in 381 msec 2024-11-27T07:24:24,933 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T07:24:24,933 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732692264933"}]},"ts":"1732692264933"} 2024-11-27T07:24:24,938 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-27T07:24:24,940 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T07:24:24,945 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2680 sec 2024-11-27T07:24:25,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T07:24:25,842 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-27T07:24:25,848 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c8feb56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:24:25,848 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 289954c9202d,43057,-1 for getting cluster id 2024-11-27T07:24:25,849 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T07:24:25,851 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '09f43d3f-c175-4713-a9ce-038f5695b74b' 2024-11-27T07:24:25,852 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T07:24:25,852 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "09f43d3f-c175-4713-a9ce-038f5695b74b" 2024-11-27T07:24:25,852 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@253618fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:24:25,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [289954c9202d,43057,-1] 2024-11-27T07:24:25,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T07:24:25,854 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:24:25,856 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52754, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T07:24:25,859 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67dec8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:24:25,859 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T07:24:25,862 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=289954c9202d,42117,1732692259671, seqNum=-1] 2024-11-27T07:24:25,862 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T07:24:25,865 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50030, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T07:24:25,868 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-27T07:24:25,870 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 289954c9202d,43057,1732692258941 2024-11-27T07:24:25,870 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1df1000 2024-11-27T07:24:25,874 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-27T07:24:25,878 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52768, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-27T07:24:25,885 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6432e8ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:24:25,885 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 289954c9202d,43057,-1 for getting cluster id 2024-11-27T07:24:25,885 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T07:24:25,887 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '09f43d3f-c175-4713-a9ce-038f5695b74b' 2024-11-27T07:24:25,888 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T07:24:25,888 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "09f43d3f-c175-4713-a9ce-038f5695b74b" 2024-11-27T07:24:25,888 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d21add1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:24:25,888 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [289954c9202d,43057,-1] 2024-11-27T07:24:25,889 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T07:24:25,894 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:24:25,897 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52786, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T07:24:25,899 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d6d4406, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:24:25,900 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@392f723b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:24:25,901 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 289954c9202d,43057,-1 for getting cluster id 2024-11-27T07:24:25,901 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T07:24:25,903 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '09f43d3f-c175-4713-a9ce-038f5695b74b' 2024-11-27T07:24:25,904 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T07:24:25,904 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "09f43d3f-c175-4713-a9ce-038f5695b74b" 2024-11-27T07:24:25,905 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@450047aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:24:25,905 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [289954c9202d,43057,-1] 2024-11-27T07:24:25,905 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T07:24:25,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:24:25,909 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52796, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T07:24:25,910 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5676fb0c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:24:25,914 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f0e98fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:24:25,914 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 289954c9202d,43057,-1 for getting cluster id 2024-11-27T07:24:25,914 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T07:24:25,918 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '09f43d3f-c175-4713-a9ce-038f5695b74b' 2024-11-27T07:24:25,919 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T07:24:25,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "09f43d3f-c175-4713-a9ce-038f5695b74b" 2024-11-27T07:24:25,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2544cc55, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:24:25,921 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [289954c9202d,43057,-1] 2024-11-27T07:24:25,921 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T07:24:25,924 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:24:25,926 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52802, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T07:24:25,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63181207, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:24:25,931 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69066494, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:24:25,931 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 289954c9202d,43057,-1 for getting cluster id 2024-11-27T07:24:25,932 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T07:24:25,935 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '09f43d3f-c175-4713-a9ce-038f5695b74b' 2024-11-27T07:24:25,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T07:24:25,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "09f43d3f-c175-4713-a9ce-038f5695b74b" 2024-11-27T07:24:25,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d22a118, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:24:25,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [289954c9202d,43057,-1] 2024-11-27T07:24:25,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T07:24:25,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:24:25,944 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52818, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T07:24:25,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8e0b13b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:24:25,948 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f00f692, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:24:25,948 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 289954c9202d,43057,-1 for getting cluster id 2024-11-27T07:24:25,948 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T07:24:25,950 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '09f43d3f-c175-4713-a9ce-038f5695b74b' 2024-11-27T07:24:25,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T07:24:25,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "09f43d3f-c175-4713-a9ce-038f5695b74b" 2024-11-27T07:24:25,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4098c4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:24:25,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [289954c9202d,43057,-1] 2024-11-27T07:24:25,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T07:24:25,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:24:25,954 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52836, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T07:24:25,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56b466e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:24:25,958 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9ec8db7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:24:25,958 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 289954c9202d,43057,-1 for getting cluster id 2024-11-27T07:24:25,959 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T07:24:25,960 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '09f43d3f-c175-4713-a9ce-038f5695b74b' 2024-11-27T07:24:25,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T07:24:25,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "09f43d3f-c175-4713-a9ce-038f5695b74b" 2024-11-27T07:24:25,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f1b5c7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:24:25,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [289954c9202d,43057,-1] 2024-11-27T07:24:25,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T07:24:25,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:24:25,965 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52848, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T07:24:25,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24bb446c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:24:25,972 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ff89310, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:24:25,973 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 289954c9202d,43057,-1 for getting cluster id 2024-11-27T07:24:25,973 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T07:24:25,975 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '09f43d3f-c175-4713-a9ce-038f5695b74b' 2024-11-27T07:24:25,975 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T07:24:25,975 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "09f43d3f-c175-4713-a9ce-038f5695b74b" 2024-11-27T07:24:25,976 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74deee7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:24:25,976 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [289954c9202d,43057,-1] 2024-11-27T07:24:25,976 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T07:24:25,977 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:24:25,978 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52860, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T07:24:25,980 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e021fe2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:24:25,981 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14d81b23, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:24:25,981 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 289954c9202d,43057,-1 for getting cluster id 2024-11-27T07:24:25,982 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T07:24:25,983 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '09f43d3f-c175-4713-a9ce-038f5695b74b' 2024-11-27T07:24:25,983 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T07:24:25,984 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "09f43d3f-c175-4713-a9ce-038f5695b74b" 2024-11-27T07:24:25,984 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@905b4d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:24:25,984 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [289954c9202d,43057,-1] 2024-11-27T07:24:25,985 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T07:24:25,985 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:24:25,986 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52886, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T07:24:25,988 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@526369d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:24:25,993 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2019371a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:24:25,993 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 289954c9202d,43057,-1 for getting cluster id 2024-11-27T07:24:25,993 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T07:24:25,994 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '09f43d3f-c175-4713-a9ce-038f5695b74b' 2024-11-27T07:24:25,995 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T07:24:25,995 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "09f43d3f-c175-4713-a9ce-038f5695b74b" 2024-11-27T07:24:25,995 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12ce558b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:24:25,996 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [289954c9202d,43057,-1] 2024-11-27T07:24:25,996 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T07:24:25,996 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:24:25,998 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52896, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T07:24:25,999 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@280a2bc3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:24:26,006 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T07:24:26,007 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T07:24:26,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=289954c9202d,42117,1732692259671, seqNum=-1] 2024-11-27T07:24:26,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T07:24:26,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=289954c9202d,42117,1732692259671, seqNum=-1] 2024-11-27T07:24:26,012 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T07:24:26,013 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50038, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T07:24:26,020 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50048, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T07:24:26,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2] 2024-11-27T07:24:26,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2] 2024-11-27T07:24:26,026 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T07:24:26,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-27T07:24:26,039 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T07:24:26,041 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T07:24:26,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=289954c9202d,42117,1732692259671, seqNum=-1] 2024-11-27T07:24:26,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T07:24:26,043 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50064, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T07:24:26,044 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T07:24:26,046 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T07:24:26,047 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T07:24:26,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2] 2024-11-27T07:24:26,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=289954c9202d,42117,1732692259671, seqNum=-1] 2024-11-27T07:24:26,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T07:24:26,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-27T07:24:26,056 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50078, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T07:24:26,061 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2] 2024-11-27T07:24:26,073 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T07:24:26,075 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=289954c9202d,42117,1732692259671, seqNum=-1] 2024-11-27T07:24:26,075 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T07:24:26,076 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T07:24:26,078 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=289954c9202d,42117,1732692259671, seqNum=-1] 2024-11-27T07:24:26,078 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T07:24:26,078 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50086, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T07:24:26,081 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50096, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T07:24:26,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2] 2024-11-27T07:24:26,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2] 2024-11-27T07:24:26,093 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T07:24:26,095 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=289954c9202d,42117,1732692259671, seqNum=-1] 2024-11-27T07:24:26,095 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T07:24:26,096 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T07:24:26,097 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50100, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T07:24:26,098 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=289954c9202d,42117,1732692259671, seqNum=-1] 2024-11-27T07:24:26,099 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T07:24:26,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2] 2024-11-27T07:24:26,101 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50102, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T07:24:26,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2] 2024-11-27T07:24:26,124 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T07:24:26,124 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T07:24:26,126 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=289954c9202d,42117,1732692259671, seqNum=-1] 2024-11-27T07:24:26,127 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T07:24:26,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 18c429ab939f89ba8ff6429b6bcb0851 2024-11-27T07:24:26,129 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 18c429ab939f89ba8ff6429b6bcb0851 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-27T07:24:26,129 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50118, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T07:24:26,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2] 2024-11-27T07:24:26,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-27T07:24:26,235 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T07:24:26,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:26,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. as already flushing 2024-11-27T07:24:26,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:26,239 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:26,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:26,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:26,294 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/bb61adad26b6438ba077c01ef4ae3ab0 is 50, key is test_row_0/A:col10/1732692266098/Put/seqid=0 2024-11-27T07:24:26,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741837_1013 (size=12001) 2024-11-27T07:24:26,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-27T07:24:26,407 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T07:24:26,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:26,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. as already flushing 2024-11-27T07:24:26,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:26,420 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:26,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:26,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:26,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:26,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:26,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50064 deadline: 1732692326425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:26,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50118 deadline: 1732692326418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:26,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:26,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:26,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50078 deadline: 1732692326443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:26,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50096 deadline: 1732692326446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:26,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:26,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50038 deadline: 1732692326452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:26,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:26,483 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:26,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:26,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:26,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:26,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:26,484 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:26,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:26,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:26,484 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:26,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:26,484 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:26,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:26,486 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:26,486 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:26,577 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T07:24:26,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:26,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. as already flushing 2024-11-27T07:24:26,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:26,582 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:26,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:26,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:26,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:26,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50064 deadline: 1732692326591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:26,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:26,595 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:26,595 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:26,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:26,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50078 deadline: 1732692326591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:26,597 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:26,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:26,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:26,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:26,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:26,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50038 deadline: 1732692326592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:26,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50118 deadline: 1732692326593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:26,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:26,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50096 deadline: 1732692326592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:26,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:26,599 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:26,600 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:26,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:26,600 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:26,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:26,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:26,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:26,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:26,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-27T07:24:26,713 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/bb61adad26b6438ba077c01ef4ae3ab0 2024-11-27T07:24:26,748 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T07:24:26,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:26,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. as already flushing 2024-11-27T07:24:26,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:26,748 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:26,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:26,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:26,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:26,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50038 deadline: 1732692326811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:26,817 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:26,818 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:26,818 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:26,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:26,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50078 deadline: 1732692326810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:26,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:26,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:26,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:26,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:26,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50096 deadline: 1732692326812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:26,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:26,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:26,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:26,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:26,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50118 deadline: 1732692326812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:26,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:26,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50064 deadline: 1732692326800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:26,822 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:26,822 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:26,822 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:26,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:26,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:26,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:26,851 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/37ad452ef6c240f49b3d5a2538072886 is 50, key is test_row_0/B:col10/1732692266098/Put/seqid=0 2024-11-27T07:24:26,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741838_1014 (size=12001) 2024-11-27T07:24:26,906 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T07:24:26,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:26,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. as already flushing 2024-11-27T07:24:26,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:26,907 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:26,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:26,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:27,062 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T07:24:27,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:27,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. as already flushing 2024-11-27T07:24:27,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:27,063 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:27,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:27,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:27,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:27,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50038 deadline: 1732692327130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:27,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:27,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:27,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:27,134 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:27,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50096 deadline: 1732692327131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:27,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:27,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50078 deadline: 1732692327131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:27,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:27,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:27,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:27,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:27,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:27,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:27,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:27,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50064 deadline: 1732692327132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:27,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:27,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:27,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:27,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:27,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50118 deadline: 1732692327139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:27,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:27,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:27,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:27,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-27T07:24:27,221 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T07:24:27,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:27,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. as already flushing 2024-11-27T07:24:27,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:27,222 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:27,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:27,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:27,293 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/37ad452ef6c240f49b3d5a2538072886 2024-11-27T07:24:27,356 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/7f93e74822e24e3682298e271de742f0 is 50, key is test_row_0/C:col10/1732692266098/Put/seqid=0 2024-11-27T07:24:27,377 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T07:24:27,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:27,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. as already flushing 2024-11-27T07:24:27,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:27,378 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:27,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:27,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:27,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741839_1015 (size=12001) 2024-11-27T07:24:27,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T07:24:27,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:27,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. as already flushing 2024-11-27T07:24:27,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:27,534 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:27,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:27,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:27,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:27,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:27,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50038 deadline: 1732692327640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:27,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50096 deadline: 1732692327640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:27,643 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:27,644 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:27,644 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:27,644 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:27,644 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:27,644 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:27,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:27,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50064 deadline: 1732692327650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:27,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:27,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:27,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:27,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:27,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50078 deadline: 1732692327650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:27,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:27,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50118 deadline: 1732692327650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:27,655 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:27,655 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:27,655 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:27,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:27,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:27,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:27,691 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T07:24:27,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:27,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. as already flushing 2024-11-27T07:24:27,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:27,691 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:27,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:27,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:27,800 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/7f93e74822e24e3682298e271de742f0 2024-11-27T07:24:27,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/bb61adad26b6438ba077c01ef4ae3ab0 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/bb61adad26b6438ba077c01ef4ae3ab0 2024-11-27T07:24:27,842 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/bb61adad26b6438ba077c01ef4ae3ab0, entries=150, sequenceid=12, filesize=11.7 K 2024-11-27T07:24:27,848 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T07:24:27,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:27,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. as already flushing 2024-11-27T07:24:27,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:27,849 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:27,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:27,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:27,860 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/37ad452ef6c240f49b3d5a2538072886 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/37ad452ef6c240f49b3d5a2538072886 2024-11-27T07:24:27,885 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/37ad452ef6c240f49b3d5a2538072886, entries=150, sequenceid=12, filesize=11.7 K 2024-11-27T07:24:27,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/7f93e74822e24e3682298e271de742f0 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/7f93e74822e24e3682298e271de742f0 2024-11-27T07:24:27,907 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/7f93e74822e24e3682298e271de742f0, entries=150, sequenceid=12, filesize=11.7 K 2024-11-27T07:24:27,918 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-27T07:24:27,921 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-27T07:24:27,928 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 18c429ab939f89ba8ff6429b6bcb0851 in 1797ms, sequenceid=12, compaction requested=false 2024-11-27T07:24:27,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 18c429ab939f89ba8ff6429b6bcb0851: 2024-11-27T07:24:28,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T07:24:28,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:28,009 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 18c429ab939f89ba8ff6429b6bcb0851 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-27T07:24:28,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/2771d35241304596af2a22bf03120038 is 50, key is test_row_0/A:col10/1732692266438/Put/seqid=0 2024-11-27T07:24:28,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741840_1016 (size=12001) 2024-11-27T07:24:28,043 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/2771d35241304596af2a22bf03120038 2024-11-27T07:24:28,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/11707fad122c46a6a3348039f9f20a82 is 50, key is test_row_0/B:col10/1732692266438/Put/seqid=0 2024-11-27T07:24:28,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741841_1017 (size=12001) 2024-11-27T07:24:28,142 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/11707fad122c46a6a3348039f9f20a82 2024-11-27T07:24:28,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/8313105513aa44379a12b308b31cb3ff is 50, key is test_row_0/C:col10/1732692266438/Put/seqid=0 2024-11-27T07:24:28,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-27T07:24:28,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741842_1018 (size=12001) 2024-11-27T07:24:28,226 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/8313105513aa44379a12b308b31cb3ff 2024-11-27T07:24:28,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/2771d35241304596af2a22bf03120038 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/2771d35241304596af2a22bf03120038 2024-11-27T07:24:28,271 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/2771d35241304596af2a22bf03120038, entries=150, sequenceid=38, filesize=11.7 K 2024-11-27T07:24:28,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/11707fad122c46a6a3348039f9f20a82 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/11707fad122c46a6a3348039f9f20a82 2024-11-27T07:24:28,295 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/11707fad122c46a6a3348039f9f20a82, entries=150, sequenceid=38, filesize=11.7 K 2024-11-27T07:24:28,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/8313105513aa44379a12b308b31cb3ff as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/8313105513aa44379a12b308b31cb3ff 2024-11-27T07:24:28,324 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/8313105513aa44379a12b308b31cb3ff, entries=150, sequenceid=38, filesize=11.7 K 2024-11-27T07:24:28,326 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=0 B/0 for 18c429ab939f89ba8ff6429b6bcb0851 in 317ms, sequenceid=38, compaction requested=false 2024-11-27T07:24:28,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 18c429ab939f89ba8ff6429b6bcb0851: 2024-11-27T07:24:28,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:28,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-27T07:24:28,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-27T07:24:28,335 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-27T07:24:28,335 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2840 sec 2024-11-27T07:24:28,341 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.3080 sec 2024-11-27T07:24:28,701 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T07:24:28,701 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 18c429ab939f89ba8ff6429b6bcb0851 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-27T07:24:28,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 18c429ab939f89ba8ff6429b6bcb0851 2024-11-27T07:24:28,721 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T07:24:28,730 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/df79fcd76ea3485980a36cd1cd00c119 is 50, key is test_row_0/A:col10/1732692268691/Put/seqid=0 2024-11-27T07:24:28,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741843_1019 (size=12001) 2024-11-27T07:24:28,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:28,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50064 deadline: 1732692328819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:28,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:28,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:28,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:28,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:28,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50118 deadline: 1732692328820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:28,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:28,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:28,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:28,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:28,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50038 deadline: 1732692328822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:28,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:28,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:28,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:28,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:28,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:28,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50078 deadline: 1732692328837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:28,861 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:28,861 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:28,861 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:28,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50096 deadline: 1732692328837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:28,873 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:28,873 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:28,873 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:28,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:28,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50118 deadline: 1732692328952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:28,959 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:28,959 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:28,959 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:28,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:28,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50064 deadline: 1732692328952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:28,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:28,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50038 deadline: 1732692328950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:28,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:28,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:28,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:28,968 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:28,968 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:28,969 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:28,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:28,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50078 deadline: 1732692328970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:28,976 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:28,976 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:28,976 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:28,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:28,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50096 deadline: 1732692328980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:28,982 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:28,982 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:28,982 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:29,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:29,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50118 deadline: 1732692329170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:29,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:29,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:29,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:29,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:29,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50038 deadline: 1732692329170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:29,174 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:29,174 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:29,174 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:29,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:29,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50064 deadline: 1732692329179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:29,186 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:29,187 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:29,187 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:29,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:29,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50078 deadline: 1732692329180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:29,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:29,189 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:29,189 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:29,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:29,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50096 deadline: 1732692329190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:29,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:29,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:29,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:29,201 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/df79fcd76ea3485980a36cd1cd00c119 2024-11-27T07:24:29,239 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/e3d25f8da2374a64b547a7528eecf9eb is 50, key is test_row_0/B:col10/1732692268691/Put/seqid=0 2024-11-27T07:24:29,270 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-27T07:24:29,271 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-27T07:24:29,274 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-27T07:24:29,274 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-27T07:24:29,274 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-27T07:24:29,274 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-27T07:24:29,276 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-27T07:24:29,276 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-27T07:24:29,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741844_1020 (size=12001) 2024-11-27T07:24:29,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:29,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50118 deadline: 1732692329480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:29,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:29,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50038 deadline: 1732692329480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:29,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:29,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:29,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:29,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:29,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:29,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:29,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:29,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50078 deadline: 1732692329500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:29,504 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:29,504 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:29,504 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:29,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:29,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50064 deadline: 1732692329500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:29,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:29,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:29,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:29,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:29,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50096 deadline: 1732692329500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:29,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:29,506 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:29,506 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:29,710 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/e3d25f8da2374a64b547a7528eecf9eb 2024-11-27T07:24:29,732 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/d9b06e2cf8814d659d91071e5517906e is 50, key is test_row_0/C:col10/1732692268691/Put/seqid=0 2024-11-27T07:24:29,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741845_1021 (size=12001) 2024-11-27T07:24:29,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:29,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50038 deadline: 1732692329989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:29,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:29,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:29,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:29,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:29,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50118 deadline: 1732692329989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:29,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:29,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:29,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:30,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:30,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50078 deadline: 1732692330010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:30,012 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:30,012 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:30,012 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:30,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:30,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50096 deadline: 1732692330011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:30,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:30,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:30,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:30,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:30,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50064 deadline: 1732692330019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:30,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:30,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:30,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:30,162 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/d9b06e2cf8814d659d91071e5517906e 2024-11-27T07:24:30,191 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/df79fcd76ea3485980a36cd1cd00c119 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/df79fcd76ea3485980a36cd1cd00c119 2024-11-27T07:24:30,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-27T07:24:30,213 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T07:24:30,214 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T07:24:30,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-27T07:24:30,218 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/df79fcd76ea3485980a36cd1cd00c119, entries=150, sequenceid=50, filesize=11.7 K 2024-11-27T07:24:30,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-27T07:24:30,219 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T07:24:30,220 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T07:24:30,220 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T07:24:30,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/e3d25f8da2374a64b547a7528eecf9eb as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/e3d25f8da2374a64b547a7528eecf9eb 2024-11-27T07:24:30,239 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/e3d25f8da2374a64b547a7528eecf9eb, entries=150, sequenceid=50, filesize=11.7 K 2024-11-27T07:24:30,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/d9b06e2cf8814d659d91071e5517906e as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/d9b06e2cf8814d659d91071e5517906e 2024-11-27T07:24:30,260 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/d9b06e2cf8814d659d91071e5517906e, entries=150, sequenceid=50, filesize=11.7 K 2024-11-27T07:24:30,263 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 18c429ab939f89ba8ff6429b6bcb0851 in 1562ms, sequenceid=50, compaction requested=true 2024-11-27T07:24:30,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 18c429ab939f89ba8ff6429b6bcb0851: 2024-11-27T07:24:30,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18c429ab939f89ba8ff6429b6bcb0851:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T07:24:30,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:24:30,270 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:24:30,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18c429ab939f89ba8ff6429b6bcb0851:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T07:24:30,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:24:30,271 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:24:30,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18c429ab939f89ba8ff6429b6bcb0851:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T07:24:30,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:24:30,274 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:24:30,274 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:24:30,276 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1541): 18c429ab939f89ba8ff6429b6bcb0851/A is initiating minor compaction (all files) 2024-11-27T07:24:30,276 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18c429ab939f89ba8ff6429b6bcb0851/A in TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:30,277 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/bb61adad26b6438ba077c01ef4ae3ab0, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/2771d35241304596af2a22bf03120038, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/df79fcd76ea3485980a36cd1cd00c119] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp, totalSize=35.2 K 2024-11-27T07:24:30,279 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb61adad26b6438ba077c01ef4ae3ab0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732692266092 2024-11-27T07:24:30,281 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2771d35241304596af2a22bf03120038, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732692266435 2024-11-27T07:24:30,282 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting df79fcd76ea3485980a36cd1cd00c119, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732692268691 2024-11-27T07:24:30,284 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 18c429ab939f89ba8ff6429b6bcb0851/B is initiating minor compaction (all files) 2024-11-27T07:24:30,284 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18c429ab939f89ba8ff6429b6bcb0851/B in TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:30,284 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/37ad452ef6c240f49b3d5a2538072886, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/11707fad122c46a6a3348039f9f20a82, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/e3d25f8da2374a64b547a7528eecf9eb] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp, totalSize=35.2 K 2024-11-27T07:24:30,288 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 37ad452ef6c240f49b3d5a2538072886, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732692266092 2024-11-27T07:24:30,289 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 11707fad122c46a6a3348039f9f20a82, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732692266435 2024-11-27T07:24:30,290 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting e3d25f8da2374a64b547a7528eecf9eb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732692268691 2024-11-27T07:24:30,328 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18c429ab939f89ba8ff6429b6bcb0851#A#compaction#9 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:24:30,329 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/e2ab7e3672db45c1b6bef1d9e2ae420c is 50, key is test_row_0/A:col10/1732692268691/Put/seqid=0 2024-11-27T07:24:30,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-27T07:24:30,336 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18c429ab939f89ba8ff6429b6bcb0851#B#compaction#10 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:24:30,338 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/870d9ccc93a74fccb6a29016d70b2663 is 50, key is test_row_0/B:col10/1732692268691/Put/seqid=0 2024-11-27T07:24:30,376 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-27T07:24:30,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:30,376 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 18c429ab939f89ba8ff6429b6bcb0851 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-27T07:24:30,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741846_1022 (size=12104) 2024-11-27T07:24:30,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741847_1023 (size=12104) 2024-11-27T07:24:30,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/235d772a54ef495d856e72d2f4a2314d is 50, key is test_row_0/A:col10/1732692268833/Put/seqid=0 2024-11-27T07:24:30,421 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/870d9ccc93a74fccb6a29016d70b2663 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/870d9ccc93a74fccb6a29016d70b2663 2024-11-27T07:24:30,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741848_1024 (size=12001) 2024-11-27T07:24:30,478 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/235d772a54ef495d856e72d2f4a2314d 2024-11-27T07:24:30,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/a27360316ddf4cb6a03a2386a0327527 is 50, key is test_row_0/B:col10/1732692268833/Put/seqid=0 2024-11-27T07:24:30,521 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 18c429ab939f89ba8ff6429b6bcb0851/B of 18c429ab939f89ba8ff6429b6bcb0851 into 870d9ccc93a74fccb6a29016d70b2663(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:24:30,521 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18c429ab939f89ba8ff6429b6bcb0851: 2024-11-27T07:24:30,525 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., storeName=18c429ab939f89ba8ff6429b6bcb0851/B, priority=13, startTime=1732692270270; duration=0sec 2024-11-27T07:24:30,525 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:24:30,526 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18c429ab939f89ba8ff6429b6bcb0851:B 2024-11-27T07:24:30,526 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:24:30,538 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:24:30,538 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 18c429ab939f89ba8ff6429b6bcb0851/C is initiating minor compaction (all files) 2024-11-27T07:24:30,538 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18c429ab939f89ba8ff6429b6bcb0851/C in TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:30,538 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/7f93e74822e24e3682298e271de742f0, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/8313105513aa44379a12b308b31cb3ff, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/d9b06e2cf8814d659d91071e5517906e] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp, totalSize=35.2 K 2024-11-27T07:24:30,539 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f93e74822e24e3682298e271de742f0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732692266092 2024-11-27T07:24:30,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-27T07:24:30,541 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 8313105513aa44379a12b308b31cb3ff, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732692266435 2024-11-27T07:24:30,547 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting d9b06e2cf8814d659d91071e5517906e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732692268691 2024-11-27T07:24:30,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741849_1025 (size=12001) 2024-11-27T07:24:30,562 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/a27360316ddf4cb6a03a2386a0327527 2024-11-27T07:24:30,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/0c8d7c2dddcb4b14ac6b9225b6960abc is 50, key is test_row_0/C:col10/1732692268833/Put/seqid=0 2024-11-27T07:24:30,604 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18c429ab939f89ba8ff6429b6bcb0851#C#compaction#14 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:24:30,605 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/a972dcd16d4e46128b3cbc3c8dfbeed4 is 50, key is test_row_0/C:col10/1732692268691/Put/seqid=0 2024-11-27T07:24:30,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741851_1027 (size=12104) 2024-11-27T07:24:30,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741850_1026 (size=12001) 2024-11-27T07:24:30,653 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/0c8d7c2dddcb4b14ac6b9225b6960abc 2024-11-27T07:24:30,669 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/a972dcd16d4e46128b3cbc3c8dfbeed4 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/a972dcd16d4e46128b3cbc3c8dfbeed4 2024-11-27T07:24:30,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/235d772a54ef495d856e72d2f4a2314d as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/235d772a54ef495d856e72d2f4a2314d 2024-11-27T07:24:30,692 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/235d772a54ef495d856e72d2f4a2314d, entries=150, sequenceid=75, filesize=11.7 K 2024-11-27T07:24:30,693 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 18c429ab939f89ba8ff6429b6bcb0851/C of 18c429ab939f89ba8ff6429b6bcb0851 into a972dcd16d4e46128b3cbc3c8dfbeed4(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:24:30,693 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18c429ab939f89ba8ff6429b6bcb0851: 2024-11-27T07:24:30,693 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., storeName=18c429ab939f89ba8ff6429b6bcb0851/C, priority=13, startTime=1732692270271; duration=0sec 2024-11-27T07:24:30,694 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:24:30,694 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18c429ab939f89ba8ff6429b6bcb0851:C 2024-11-27T07:24:30,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/a27360316ddf4cb6a03a2386a0327527 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/a27360316ddf4cb6a03a2386a0327527 2024-11-27T07:24:30,707 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/a27360316ddf4cb6a03a2386a0327527, entries=150, sequenceid=75, filesize=11.7 K 2024-11-27T07:24:30,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/0c8d7c2dddcb4b14ac6b9225b6960abc as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/0c8d7c2dddcb4b14ac6b9225b6960abc 2024-11-27T07:24:30,732 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/0c8d7c2dddcb4b14ac6b9225b6960abc, entries=150, sequenceid=75, filesize=11.7 K 2024-11-27T07:24:30,735 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=0 B/0 for 18c429ab939f89ba8ff6429b6bcb0851 in 359ms, sequenceid=75, compaction requested=false 2024-11-27T07:24:30,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 18c429ab939f89ba8ff6429b6bcb0851: 2024-11-27T07:24:30,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:30,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-27T07:24:30,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-27T07:24:30,751 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-27T07:24:30,751 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 520 msec 2024-11-27T07:24:30,758 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 539 msec 2024-11-27T07:24:30,806 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/e2ab7e3672db45c1b6bef1d9e2ae420c as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/e2ab7e3672db45c1b6bef1d9e2ae420c 2024-11-27T07:24:30,831 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 18c429ab939f89ba8ff6429b6bcb0851/A of 18c429ab939f89ba8ff6429b6bcb0851 into e2ab7e3672db45c1b6bef1d9e2ae420c(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:24:30,831 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18c429ab939f89ba8ff6429b6bcb0851: 2024-11-27T07:24:30,831 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., storeName=18c429ab939f89ba8ff6429b6bcb0851/A, priority=13, startTime=1732692270266; duration=0sec 2024-11-27T07:24:30,831 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:24:30,831 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18c429ab939f89ba8ff6429b6bcb0851:A 2024-11-27T07:24:30,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-27T07:24:30,850 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T07:24:30,851 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T07:24:30,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-27T07:24:30,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-27T07:24:30,855 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T07:24:30,856 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T07:24:30,857 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T07:24:30,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-27T07:24:31,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-27T07:24:31,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:31,013 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 18c429ab939f89ba8ff6429b6bcb0851 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-11-27T07:24:31,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/8ec64fce945d4655b8c2d178adc1ecc7 is 50, key is test_row_0/A:col10/1732692271009/Put/seqid=0 2024-11-27T07:24:31,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 18c429ab939f89ba8ff6429b6bcb0851 2024-11-27T07:24:31,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. as already flushing 2024-11-27T07:24:31,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741852_1028 (size=12001) 2024-11-27T07:24:31,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:31,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50064 deadline: 1732692331114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:31,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:31,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:31,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:31,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:31,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50118 deadline: 1732692331119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:31,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:31,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:31,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:31,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:31,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50096 deadline: 1732692331124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:31,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:31,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:31,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:31,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50078 deadline: 1732692331126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:31,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:31,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:31,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:31,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:31,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:31,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50038 deadline: 1732692331132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:31,141 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:31,141 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:31,141 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:31,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-27T07:24:31,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:31,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50064 deadline: 1732692331239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:31,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:31,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:31,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:31,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:31,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50118 deadline: 1732692331240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:31,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:31,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:31,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:31,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:31,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50078 deadline: 1732692331240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:31,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:31,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50096 deadline: 1732692331240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:31,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:31,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:31,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:31,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:31,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:31,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:31,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:31,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50038 deadline: 1732692331249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:31,259 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:31,259 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:31,259 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:31,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:31,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50064 deadline: 1732692331450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:31,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:31,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50118 deadline: 1732692331450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:31,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:31,456 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:31,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:31,456 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:31,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:31,456 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:31,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:31,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50096 deadline: 1732692331460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:31,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:31,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50078 deadline: 1732692331460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:31,463 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:31,463 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:31,463 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:31,465 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:31,465 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:31,465 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:31,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:31,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50038 deadline: 1732692331470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:31,474 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:31,474 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:31,474 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:31,477 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/8ec64fce945d4655b8c2d178adc1ecc7 2024-11-27T07:24:31,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-27T07:24:31,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/37723e1c6944487f9d8953338db3fbc5 is 50, key is test_row_0/B:col10/1732692271009/Put/seqid=0 2024-11-27T07:24:31,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741853_1029 (size=12001) 2024-11-27T07:24:31,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:31,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50064 deadline: 1732692331759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:31,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:31,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:31,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:31,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:31,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50118 deadline: 1732692331769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:31,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:31,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50078 deadline: 1732692331770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:31,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:31,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50096 deadline: 1732692331770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:31,775 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:31,776 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:31,776 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:31,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:31,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:31,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:31,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:31,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:31,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:31,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:31,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50038 deadline: 1732692331780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:31,781 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:31,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:31,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:31,947 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/37723e1c6944487f9d8953338db3fbc5 2024-11-27T07:24:31,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/d76d5ff811084c188c00a95f0a1d4033 is 50, key is test_row_0/C:col10/1732692271009/Put/seqid=0 2024-11-27T07:24:31,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741854_1030 (size=12001) 2024-11-27T07:24:31,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-27T07:24:32,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:32,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50064 deadline: 1732692332269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:32,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:32,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:32,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:32,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:32,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50078 deadline: 1732692332279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:32,284 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:32,284 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:32,284 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:32,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:32,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:32,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50118 deadline: 1732692332289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:32,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50096 deadline: 1732692332289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:32,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:32,296 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:32,296 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:32,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:32,296 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:32,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:32,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:32,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50038 deadline: 1732692332289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:32,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:32,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:32,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:32,380 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/d76d5ff811084c188c00a95f0a1d4033 2024-11-27T07:24:32,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/8ec64fce945d4655b8c2d178adc1ecc7 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/8ec64fce945d4655b8c2d178adc1ecc7 2024-11-27T07:24:32,412 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/8ec64fce945d4655b8c2d178adc1ecc7, entries=150, sequenceid=84, filesize=11.7 K 2024-11-27T07:24:32,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/37723e1c6944487f9d8953338db3fbc5 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/37723e1c6944487f9d8953338db3fbc5 2024-11-27T07:24:32,428 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/37723e1c6944487f9d8953338db3fbc5, entries=150, sequenceid=84, filesize=11.7 K 2024-11-27T07:24:32,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/d76d5ff811084c188c00a95f0a1d4033 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/d76d5ff811084c188c00a95f0a1d4033 2024-11-27T07:24:32,449 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/d76d5ff811084c188c00a95f0a1d4033, entries=150, sequenceid=84, filesize=11.7 K 2024-11-27T07:24:32,452 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=194.56 KB/199230 for 18c429ab939f89ba8ff6429b6bcb0851 in 1439ms, sequenceid=84, compaction requested=true 2024-11-27T07:24:32,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 18c429ab939f89ba8ff6429b6bcb0851: 2024-11-27T07:24:32,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:32,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-27T07:24:32,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-27T07:24:32,473 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-27T07:24:32,473 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6090 sec 2024-11-27T07:24:32,478 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.6240 sec 2024-11-27T07:24:33,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-27T07:24:33,010 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T07:24:33,011 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T07:24:33,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-27T07:24:33,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-27T07:24:33,014 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T07:24:33,015 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T07:24:33,016 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T07:24:33,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-27T07:24:33,173 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-27T07:24:33,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:33,173 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 18c429ab939f89ba8ff6429b6bcb0851 3/3 column families, dataSize=194.56 KB heapSize=510.52 KB 2024-11-27T07:24:33,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/0771928f1e1842b6a9fd37fbca6d184c is 50, key is test_row_0/A:col10/1732692271120/Put/seqid=0 2024-11-27T07:24:33,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741855_1031 (size=12001) 2024-11-27T07:24:33,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 18c429ab939f89ba8ff6429b6bcb0851 2024-11-27T07:24:33,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. as already flushing 2024-11-27T07:24:33,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:33,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50064 deadline: 1732692333293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:33,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:33,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:33,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:33,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:33,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:33,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50078 deadline: 1732692333299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:33,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50096 deadline: 1732692333299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:33,305 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 289954c9202d,42117,1732692259671 for 'test_row_1' in 18c429ab939f89ba8ff6429b6bcb0851 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T07:24:33,305 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 289954c9202d,42117,1732692259671 for 'test_row_0' in 18c429ab939f89ba8ff6429b6bcb0851 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T07:24:33,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:33,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:33,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:33,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:33,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:33,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:33,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:33,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50118 deadline: 1732692333309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:33,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:33,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50038 deadline: 1732692333309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:33,311 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 289954c9202d,42117,1732692259671 for 'test_row_2' in 18c429ab939f89ba8ff6429b6bcb0851 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T07:24:33,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:33,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:33,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:33,312 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 289954c9202d,42117,1732692259671 for 'test_row_1' in 18c429ab939f89ba8ff6429b6bcb0851 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T07:24:33,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:33,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:33,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:33,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-27T07:24:33,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:33,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50064 deadline: 1732692333409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:33,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:33,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:33,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:33,605 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/0771928f1e1842b6a9fd37fbca6d184c 2024-11-27T07:24:33,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:33,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50064 deadline: 1732692333619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:33,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:33,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:33,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:33,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-27T07:24:33,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/b49eb9d3ba73439aa95c84f989b57e92 is 50, key is test_row_0/B:col10/1732692271120/Put/seqid=0 2024-11-27T07:24:33,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741856_1032 (size=12001) 2024-11-27T07:24:33,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:33,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50064 deadline: 1732692333929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:33,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:33,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:33,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:34,063 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/b49eb9d3ba73439aa95c84f989b57e92 2024-11-27T07:24:34,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/a675df890e544088b436b089046d33d7 is 50, key is test_row_0/C:col10/1732692271120/Put/seqid=0 2024-11-27T07:24:34,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741857_1033 (size=12001) 2024-11-27T07:24:34,129 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/a675df890e544088b436b089046d33d7 2024-11-27T07:24:34,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/0771928f1e1842b6a9fd37fbca6d184c as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/0771928f1e1842b6a9fd37fbca6d184c 2024-11-27T07:24:34,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-27T07:24:34,166 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/0771928f1e1842b6a9fd37fbca6d184c, entries=150, sequenceid=116, filesize=11.7 K 2024-11-27T07:24:34,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/b49eb9d3ba73439aa95c84f989b57e92 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/b49eb9d3ba73439aa95c84f989b57e92 2024-11-27T07:24:34,180 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/b49eb9d3ba73439aa95c84f989b57e92, entries=150, sequenceid=116, filesize=11.7 K 2024-11-27T07:24:34,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/a675df890e544088b436b089046d33d7 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/a675df890e544088b436b089046d33d7 2024-11-27T07:24:34,202 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/a675df890e544088b436b089046d33d7, entries=150, sequenceid=116, filesize=11.7 K 2024-11-27T07:24:34,203 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~194.56 KB/199230, heapSize ~510.47 KB/522720, currentSize=6.71 KB/6870 for 18c429ab939f89ba8ff6429b6bcb0851 in 1030ms, sequenceid=116, compaction requested=true 2024-11-27T07:24:34,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 18c429ab939f89ba8ff6429b6bcb0851: 2024-11-27T07:24:34,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:34,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-27T07:24:34,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-27T07:24:34,212 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-27T07:24:34,213 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1910 sec 2024-11-27T07:24:34,217 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.2030 sec 2024-11-27T07:24:34,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 18c429ab939f89ba8ff6429b6bcb0851 2024-11-27T07:24:34,480 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T07:24:34,481 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 18c429ab939f89ba8ff6429b6bcb0851 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-27T07:24:34,487 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/9a8b745bc9894c61b15cc8d334a06060 is 50, key is test_row_0/A:col10/1732692274471/Put/seqid=0 2024-11-27T07:24:34,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741858_1034 (size=12001) 2024-11-27T07:24:34,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:34,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50064 deadline: 1732692334618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:34,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:34,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:34,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:34,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:34,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50064 deadline: 1732692334730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:34,731 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:34,731 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:34,731 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:34,900 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/9a8b745bc9894c61b15cc8d334a06060 2024-11-27T07:24:34,942 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/42b329dea6a543c28685aca7724a457d is 50, key is test_row_0/B:col10/1732692274471/Put/seqid=0 2024-11-27T07:24:34,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:34,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50064 deadline: 1732692334940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:34,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:34,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:34,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:34,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741859_1035 (size=12001) 2024-11-27T07:24:34,961 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/42b329dea6a543c28685aca7724a457d 2024-11-27T07:24:34,997 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/e96dc4b20bd84f098d5ff8d3599f6859 is 50, key is test_row_0/C:col10/1732692274471/Put/seqid=0 2024-11-27T07:24:35,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741860_1036 (size=12001) 2024-11-27T07:24:35,015 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/e96dc4b20bd84f098d5ff8d3599f6859 2024-11-27T07:24:35,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/9a8b745bc9894c61b15cc8d334a06060 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/9a8b745bc9894c61b15cc8d334a06060 2024-11-27T07:24:35,047 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/9a8b745bc9894c61b15cc8d334a06060, entries=150, sequenceid=127, filesize=11.7 K 2024-11-27T07:24:35,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/42b329dea6a543c28685aca7724a457d as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/42b329dea6a543c28685aca7724a457d 2024-11-27T07:24:35,061 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/42b329dea6a543c28685aca7724a457d, entries=150, sequenceid=127, filesize=11.7 K 2024-11-27T07:24:35,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/e96dc4b20bd84f098d5ff8d3599f6859 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/e96dc4b20bd84f098d5ff8d3599f6859 2024-11-27T07:24:35,081 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/e96dc4b20bd84f098d5ff8d3599f6859, entries=150, sequenceid=127, filesize=11.7 K 2024-11-27T07:24:35,090 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 18c429ab939f89ba8ff6429b6bcb0851 in 609ms, sequenceid=127, compaction requested=true 2024-11-27T07:24:35,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 18c429ab939f89ba8ff6429b6bcb0851: 2024-11-27T07:24:35,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18c429ab939f89ba8ff6429b6bcb0851:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T07:24:35,091 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-27T07:24:35,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:24:35,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18c429ab939f89ba8ff6429b6bcb0851:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T07:24:35,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:24:35,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18c429ab939f89ba8ff6429b6bcb0851:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T07:24:35,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:24:35,091 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-27T07:24:35,097 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60108 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-27T07:24:35,097 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60108 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-27T07:24:35,097 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 18c429ab939f89ba8ff6429b6bcb0851/A is initiating minor compaction (all files) 2024-11-27T07:24:35,097 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1541): 18c429ab939f89ba8ff6429b6bcb0851/B is initiating minor compaction (all files) 2024-11-27T07:24:35,097 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18c429ab939f89ba8ff6429b6bcb0851/B in TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:35,097 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18c429ab939f89ba8ff6429b6bcb0851/A in TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:35,098 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/870d9ccc93a74fccb6a29016d70b2663, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/a27360316ddf4cb6a03a2386a0327527, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/37723e1c6944487f9d8953338db3fbc5, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/b49eb9d3ba73439aa95c84f989b57e92, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/42b329dea6a543c28685aca7724a457d] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp, totalSize=58.7 K 2024-11-27T07:24:35,098 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/e2ab7e3672db45c1b6bef1d9e2ae420c, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/235d772a54ef495d856e72d2f4a2314d, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/8ec64fce945d4655b8c2d178adc1ecc7, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/0771928f1e1842b6a9fd37fbca6d184c, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/9a8b745bc9894c61b15cc8d334a06060] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp, totalSize=58.7 K 2024-11-27T07:24:35,099 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting e2ab7e3672db45c1b6bef1d9e2ae420c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732692268691 2024-11-27T07:24:35,099 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 870d9ccc93a74fccb6a29016d70b2663, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732692268691 2024-11-27T07:24:35,100 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting a27360316ddf4cb6a03a2386a0327527, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1732692268813 2024-11-27T07:24:35,101 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 235d772a54ef495d856e72d2f4a2314d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1732692268813 2024-11-27T07:24:35,102 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ec64fce945d4655b8c2d178adc1ecc7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1732692271001 2024-11-27T07:24:35,103 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37723e1c6944487f9d8953338db3fbc5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1732692271001 2024-11-27T07:24:35,103 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 0771928f1e1842b6a9fd37fbca6d184c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732692271119 2024-11-27T07:24:35,106 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting b49eb9d3ba73439aa95c84f989b57e92, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732692271119 2024-11-27T07:24:35,107 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42b329dea6a543c28685aca7724a457d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732692274471 2024-11-27T07:24:35,113 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a8b745bc9894c61b15cc8d334a06060, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732692274471 2024-11-27T07:24:35,139 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18c429ab939f89ba8ff6429b6bcb0851#B#compaction#24 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:24:35,140 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/49482dde3e574e8bb9cee3bc38557db0 is 50, key is test_row_0/B:col10/1732692274471/Put/seqid=0 2024-11-27T07:24:35,147 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18c429ab939f89ba8ff6429b6bcb0851#A#compaction#25 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:24:35,148 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/d04fac4f2bd6432e9fe7d2ce8bba4717 is 50, key is test_row_0/A:col10/1732692274471/Put/seqid=0 2024-11-27T07:24:35,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741861_1037 (size=12275) 2024-11-27T07:24:35,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-27T07:24:35,159 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T07:24:35,160 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T07:24:35,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-27T07:24:35,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-27T07:24:35,163 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T07:24:35,166 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T07:24:35,167 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T07:24:35,171 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/49482dde3e574e8bb9cee3bc38557db0 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/49482dde3e574e8bb9cee3bc38557db0 2024-11-27T07:24:35,187 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 18c429ab939f89ba8ff6429b6bcb0851/B of 18c429ab939f89ba8ff6429b6bcb0851 into 49482dde3e574e8bb9cee3bc38557db0(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:24:35,187 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18c429ab939f89ba8ff6429b6bcb0851: 2024-11-27T07:24:35,187 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., storeName=18c429ab939f89ba8ff6429b6bcb0851/B, priority=11, startTime=1732692275091; duration=0sec 2024-11-27T07:24:35,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741862_1038 (size=12275) 2024-11-27T07:24:35,204 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:24:35,205 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18c429ab939f89ba8ff6429b6bcb0851:B 2024-11-27T07:24:35,205 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-27T07:24:35,207 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60108 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-27T07:24:35,208 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1541): 18c429ab939f89ba8ff6429b6bcb0851/C is initiating minor compaction (all files) 2024-11-27T07:24:35,208 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18c429ab939f89ba8ff6429b6bcb0851/C in TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:35,208 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/a972dcd16d4e46128b3cbc3c8dfbeed4, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/0c8d7c2dddcb4b14ac6b9225b6960abc, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/d76d5ff811084c188c00a95f0a1d4033, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/a675df890e544088b436b089046d33d7, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/e96dc4b20bd84f098d5ff8d3599f6859] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp, totalSize=58.7 K 2024-11-27T07:24:35,209 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting a972dcd16d4e46128b3cbc3c8dfbeed4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732692268691 2024-11-27T07:24:35,210 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c8d7c2dddcb4b14ac6b9225b6960abc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1732692268813 2024-11-27T07:24:35,211 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting d76d5ff811084c188c00a95f0a1d4033, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1732692271001 2024-11-27T07:24:35,211 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting a675df890e544088b436b089046d33d7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732692271119 2024-11-27T07:24:35,212 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting e96dc4b20bd84f098d5ff8d3599f6859, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732692274471 2024-11-27T07:24:35,235 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18c429ab939f89ba8ff6429b6bcb0851#C#compaction#26 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:24:35,236 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/840cf06b1fbf481d89b11637ec71caef is 50, key is test_row_0/C:col10/1732692274471/Put/seqid=0 2024-11-27T07:24:35,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 18c429ab939f89ba8ff6429b6bcb0851 2024-11-27T07:24:35,251 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T07:24:35,251 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 18c429ab939f89ba8ff6429b6bcb0851 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-27T07:24:35,260 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/d51a7bb5825c4c908d13430a9fb61b09 is 50, key is test_row_0/A:col10/1732692274612/Put/seqid=0 2024-11-27T07:24:35,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-27T07:24:35,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741863_1039 (size=12275) 2024-11-27T07:24:35,294 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/840cf06b1fbf481d89b11637ec71caef as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/840cf06b1fbf481d89b11637ec71caef 2024-11-27T07:24:35,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741864_1040 (size=14541) 2024-11-27T07:24:35,297 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/d51a7bb5825c4c908d13430a9fb61b09 2024-11-27T07:24:35,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:35,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50064 deadline: 1732692335302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:35,304 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:35,304 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:35,304 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:35,313 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 18c429ab939f89ba8ff6429b6bcb0851/C of 18c429ab939f89ba8ff6429b6bcb0851 into 840cf06b1fbf481d89b11637ec71caef(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:24:35,313 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18c429ab939f89ba8ff6429b6bcb0851: 2024-11-27T07:24:35,313 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., storeName=18c429ab939f89ba8ff6429b6bcb0851/C, priority=11, startTime=1732692275091; duration=0sec 2024-11-27T07:24:35,313 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:24:35,313 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18c429ab939f89ba8ff6429b6bcb0851:C 2024-11-27T07:24:35,315 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/a1e46af630fd4680af93df8d585f7b07 is 50, key is test_row_0/B:col10/1732692274612/Put/seqid=0 2024-11-27T07:24:35,322 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-27T07:24:35,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:35,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. as already flushing 2024-11-27T07:24:35,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:35,323 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:35,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:35,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:35,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:35,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50038 deadline: 1732692335319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:35,330 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 289954c9202d,42117,1732692259671 for 'test_row_1' in 18c429ab939f89ba8ff6429b6bcb0851 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T07:24:35,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:35,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:35,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:35,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:35,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50118 deadline: 1732692335330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:35,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:35,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50078 deadline: 1732692335330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:35,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:35,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50096 deadline: 1732692335330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:35,336 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 289954c9202d,42117,1732692259671 for 'test_row_2' in 18c429ab939f89ba8ff6429b6bcb0851 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T07:24:35,336 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 289954c9202d,42117,1732692259671 for 'test_row_1' in 18c429ab939f89ba8ff6429b6bcb0851 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T07:24:35,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:35,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:35,337 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:35,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:35,337 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:35,337 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:35,337 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 289954c9202d,42117,1732692259671 for 'test_row_0' in 18c429ab939f89ba8ff6429b6bcb0851 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T07:24:35,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:35,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:35,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:35,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741865_1041 (size=12151) 2024-11-27T07:24:35,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:35,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50064 deadline: 1732692335410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:35,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:35,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:35,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:35,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-27T07:24:35,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-27T07:24:35,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:35,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. as already flushing 2024-11-27T07:24:35,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:35,480 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:35,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:35,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:35,612 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/d04fac4f2bd6432e9fe7d2ce8bba4717 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/d04fac4f2bd6432e9fe7d2ce8bba4717 2024-11-27T07:24:35,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:35,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50064 deadline: 1732692335620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:35,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:35,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:35,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:35,626 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 18c429ab939f89ba8ff6429b6bcb0851/A of 18c429ab939f89ba8ff6429b6bcb0851 into d04fac4f2bd6432e9fe7d2ce8bba4717(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:24:35,626 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18c429ab939f89ba8ff6429b6bcb0851: 2024-11-27T07:24:35,626 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., storeName=18c429ab939f89ba8ff6429b6bcb0851/A, priority=11, startTime=1732692275090; duration=0sec 2024-11-27T07:24:35,626 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:24:35,626 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18c429ab939f89ba8ff6429b6bcb0851:A 2024-11-27T07:24:35,639 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-27T07:24:35,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:35,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. as already flushing 2024-11-27T07:24:35,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:35,640 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:35,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:35,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:35,761 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/a1e46af630fd4680af93df8d585f7b07 2024-11-27T07:24:35,780 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/a00924dc2f034e94831bf9535d415198 is 50, key is test_row_0/C:col10/1732692274612/Put/seqid=0 2024-11-27T07:24:35,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-27T07:24:35,795 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-27T07:24:35,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:35,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. as already flushing 2024-11-27T07:24:35,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:35,796 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:35,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:35,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:35,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741866_1042 (size=12151) 2024-11-27T07:24:35,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:35,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50064 deadline: 1732692335929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:35,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:35,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:35,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:35,954 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-27T07:24:35,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:35,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. as already flushing 2024-11-27T07:24:35,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:35,955 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:35,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:35,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:36,110 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-27T07:24:36,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:36,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. as already flushing 2024-11-27T07:24:36,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:36,111 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:36,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:36,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:36,242 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/a00924dc2f034e94831bf9535d415198 2024-11-27T07:24:36,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/d51a7bb5825c4c908d13430a9fb61b09 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/d51a7bb5825c4c908d13430a9fb61b09 2024-11-27T07:24:36,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-27T07:24:36,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:36,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. as already flushing 2024-11-27T07:24:36,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:36,266 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:36,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:36,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:36,269 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/d51a7bb5825c4c908d13430a9fb61b09, entries=200, sequenceid=154, filesize=14.2 K 2024-11-27T07:24:36,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/a1e46af630fd4680af93df8d585f7b07 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/a1e46af630fd4680af93df8d585f7b07 2024-11-27T07:24:36,287 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/a1e46af630fd4680af93df8d585f7b07, entries=150, sequenceid=154, filesize=11.9 K 2024-11-27T07:24:36,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/a00924dc2f034e94831bf9535d415198 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/a00924dc2f034e94831bf9535d415198 2024-11-27T07:24:36,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-27T07:24:36,301 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/a00924dc2f034e94831bf9535d415198, entries=150, sequenceid=154, filesize=11.9 K 2024-11-27T07:24:36,303 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 18c429ab939f89ba8ff6429b6bcb0851 in 1051ms, sequenceid=154, compaction requested=false 2024-11-27T07:24:36,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 18c429ab939f89ba8ff6429b6bcb0851: 2024-11-27T07:24:36,422 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-27T07:24:36,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:36,422 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 18c429ab939f89ba8ff6429b6bcb0851 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-27T07:24:36,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/ecc6c9ee8cda46ed957710dd7c0d1068 is 50, key is test_row_0/A:col10/1732692275285/Put/seqid=0 2024-11-27T07:24:36,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 18c429ab939f89ba8ff6429b6bcb0851 2024-11-27T07:24:36,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741867_1043 (size=12151) 2024-11-27T07:24:36,444 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/ecc6c9ee8cda46ed957710dd7c0d1068 2024-11-27T07:24:36,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. as already flushing 2024-11-27T07:24:36,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/5488d1fe32c4420f8f62607c9b800b62 is 50, key is test_row_0/B:col10/1732692275285/Put/seqid=0 2024-11-27T07:24:36,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741868_1044 (size=12151) 2024-11-27T07:24:36,520 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/5488d1fe32c4420f8f62607c9b800b62 2024-11-27T07:24:36,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/bcf5949156f343f1ae4359ffe9cada4a is 50, key is test_row_0/C:col10/1732692275285/Put/seqid=0 2024-11-27T07:24:36,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741869_1045 (size=12151) 2024-11-27T07:24:36,610 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/bcf5949156f343f1ae4359ffe9cada4a 2024-11-27T07:24:36,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/ecc6c9ee8cda46ed957710dd7c0d1068 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/ecc6c9ee8cda46ed957710dd7c0d1068 2024-11-27T07:24:36,629 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/ecc6c9ee8cda46ed957710dd7c0d1068, entries=150, sequenceid=166, filesize=11.9 K 2024-11-27T07:24:36,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:36,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/5488d1fe32c4420f8f62607c9b800b62 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/5488d1fe32c4420f8f62607c9b800b62 2024-11-27T07:24:36,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50064 deadline: 1732692336626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:36,633 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:36,633 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:36,633 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:36,643 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/5488d1fe32c4420f8f62607c9b800b62, entries=150, sequenceid=166, filesize=11.9 K 2024-11-27T07:24:36,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/bcf5949156f343f1ae4359ffe9cada4a as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/bcf5949156f343f1ae4359ffe9cada4a 2024-11-27T07:24:36,654 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/bcf5949156f343f1ae4359ffe9cada4a, entries=150, sequenceid=166, filesize=11.9 K 2024-11-27T07:24:36,656 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 18c429ab939f89ba8ff6429b6bcb0851 in 234ms, sequenceid=166, compaction requested=true 2024-11-27T07:24:36,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 18c429ab939f89ba8ff6429b6bcb0851: 2024-11-27T07:24:36,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:36,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-27T07:24:36,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-27T07:24:36,671 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-27T07:24:36,671 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4910 sec 2024-11-27T07:24:36,675 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.5130 sec 2024-11-27T07:24:36,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 18c429ab939f89ba8ff6429b6bcb0851 2024-11-27T07:24:36,743 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T07:24:36,743 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 18c429ab939f89ba8ff6429b6bcb0851 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-27T07:24:36,759 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/87c8c972c53747a090b431f2392d7847 is 50, key is test_row_0/A:col10/1732692276566/Put/seqid=0 2024-11-27T07:24:36,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:36,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50064 deadline: 1732692336797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:36,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:36,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:36,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:36,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741870_1046 (size=14541) 2024-11-27T07:24:36,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:36,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50064 deadline: 1732692336909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:36,914 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:36,914 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:36,914 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:37,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:37,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50064 deadline: 1732692337119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:37,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:37,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:37,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:37,216 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/87c8c972c53747a090b431f2392d7847 2024-11-27T07:24:37,235 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/3d4da3171a4a4f3eb05197f69fd56b2c is 50, key is test_row_0/B:col10/1732692276566/Put/seqid=0 2024-11-27T07:24:37,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741871_1047 (size=12151) 2024-11-27T07:24:37,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-27T07:24:37,317 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T07:24:37,317 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T07:24:37,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-27T07:24:37,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-27T07:24:37,320 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T07:24:37,321 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T07:24:37,321 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T07:24:37,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-27T07:24:37,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:37,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50064 deadline: 1732692337430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:37,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:37,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:37,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:37,479 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-27T07:24:37,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:37,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. as already flushing 2024-11-27T07:24:37,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:37,479 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:37,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:37,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:37,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-27T07:24:37,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:37,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. as already flushing 2024-11-27T07:24:37,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:37,633 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:37,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:37,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:37,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-27T07:24:37,651 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/3d4da3171a4a4f3eb05197f69fd56b2c 2024-11-27T07:24:37,663 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/f39371934e4c446cbd3ce70fa5691aad is 50, key is test_row_0/C:col10/1732692276566/Put/seqid=0 2024-11-27T07:24:37,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741872_1048 (size=12151) 2024-11-27T07:24:37,680 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/f39371934e4c446cbd3ce70fa5691aad 2024-11-27T07:24:37,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/87c8c972c53747a090b431f2392d7847 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/87c8c972c53747a090b431f2392d7847 2024-11-27T07:24:37,701 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/87c8c972c53747a090b431f2392d7847, entries=200, sequenceid=193, filesize=14.2 K 2024-11-27T07:24:37,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/3d4da3171a4a4f3eb05197f69fd56b2c as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/3d4da3171a4a4f3eb05197f69fd56b2c 2024-11-27T07:24:37,710 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/3d4da3171a4a4f3eb05197f69fd56b2c, entries=150, sequenceid=193, filesize=11.9 K 2024-11-27T07:24:37,712 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/f39371934e4c446cbd3ce70fa5691aad as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/f39371934e4c446cbd3ce70fa5691aad 2024-11-27T07:24:37,724 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/f39371934e4c446cbd3ce70fa5691aad, entries=150, sequenceid=193, filesize=11.9 K 2024-11-27T07:24:37,726 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 18c429ab939f89ba8ff6429b6bcb0851 in 982ms, sequenceid=193, compaction requested=true 2024-11-27T07:24:37,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 18c429ab939f89ba8ff6429b6bcb0851: 2024-11-27T07:24:37,726 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T07:24:37,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18c429ab939f89ba8ff6429b6bcb0851:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T07:24:37,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:24:37,727 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T07:24:37,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18c429ab939f89ba8ff6429b6bcb0851:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T07:24:37,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:24:37,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18c429ab939f89ba8ff6429b6bcb0851:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T07:24:37,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:24:37,729 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53508 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T07:24:37,729 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1541): 18c429ab939f89ba8ff6429b6bcb0851/A is initiating minor compaction (all files) 2024-11-27T07:24:37,729 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18c429ab939f89ba8ff6429b6bcb0851/A in TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:37,729 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/d04fac4f2bd6432e9fe7d2ce8bba4717, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/d51a7bb5825c4c908d13430a9fb61b09, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/ecc6c9ee8cda46ed957710dd7c0d1068, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/87c8c972c53747a090b431f2392d7847] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp, totalSize=52.3 K 2024-11-27T07:24:37,730 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48728 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T07:24:37,730 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 18c429ab939f89ba8ff6429b6bcb0851/B is initiating minor compaction (all files) 2024-11-27T07:24:37,730 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18c429ab939f89ba8ff6429b6bcb0851/B in TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:37,730 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/49482dde3e574e8bb9cee3bc38557db0, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/a1e46af630fd4680af93df8d585f7b07, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/5488d1fe32c4420f8f62607c9b800b62, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/3d4da3171a4a4f3eb05197f69fd56b2c] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp, totalSize=47.6 K 2024-11-27T07:24:37,731 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting d04fac4f2bd6432e9fe7d2ce8bba4717, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732692274471 2024-11-27T07:24:37,731 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 49482dde3e574e8bb9cee3bc38557db0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732692274471 2024-11-27T07:24:37,731 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting d51a7bb5825c4c908d13430a9fb61b09, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732692274592 2024-11-27T07:24:37,732 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting a1e46af630fd4680af93df8d585f7b07, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732692274592 2024-11-27T07:24:37,732 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting ecc6c9ee8cda46ed957710dd7c0d1068, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732692275259 2024-11-27T07:24:37,733 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87c8c972c53747a090b431f2392d7847, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732692276566 2024-11-27T07:24:37,734 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 5488d1fe32c4420f8f62607c9b800b62, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732692275259 2024-11-27T07:24:37,734 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d4da3171a4a4f3eb05197f69fd56b2c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732692276566 2024-11-27T07:24:37,753 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18c429ab939f89ba8ff6429b6bcb0851#B#compaction#36 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:24:37,753 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/8c049c9492a34f37bbee7dc9a00b9f78 is 50, key is test_row_0/B:col10/1732692276566/Put/seqid=0 2024-11-27T07:24:37,756 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18c429ab939f89ba8ff6429b6bcb0851#A#compaction#37 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:24:37,757 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/30df5ea5072946f4834f08170594efa6 is 50, key is test_row_0/A:col10/1732692276566/Put/seqid=0 2024-11-27T07:24:37,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741873_1049 (size=12561) 2024-11-27T07:24:37,780 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/8c049c9492a34f37bbee7dc9a00b9f78 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/8c049c9492a34f37bbee7dc9a00b9f78 2024-11-27T07:24:37,787 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-27T07:24:37,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:37,788 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 18c429ab939f89ba8ff6429b6bcb0851 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-27T07:24:37,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/9c55ea41112440d9b90dca7ccd6c33fd is 50, key is test_row_0/A:col10/1732692276793/Put/seqid=0 2024-11-27T07:24:37,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741874_1050 (size=12561) 2024-11-27T07:24:37,799 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 18c429ab939f89ba8ff6429b6bcb0851/B of 18c429ab939f89ba8ff6429b6bcb0851 into 8c049c9492a34f37bbee7dc9a00b9f78(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:24:37,799 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18c429ab939f89ba8ff6429b6bcb0851: 2024-11-27T07:24:37,799 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., storeName=18c429ab939f89ba8ff6429b6bcb0851/B, priority=12, startTime=1732692277727; duration=0sec 2024-11-27T07:24:37,799 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:24:37,800 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18c429ab939f89ba8ff6429b6bcb0851:B 2024-11-27T07:24:37,800 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T07:24:37,803 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48728 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T07:24:37,803 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 18c429ab939f89ba8ff6429b6bcb0851/C is initiating minor compaction (all files) 2024-11-27T07:24:37,803 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18c429ab939f89ba8ff6429b6bcb0851/C in TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:37,804 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/840cf06b1fbf481d89b11637ec71caef, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/a00924dc2f034e94831bf9535d415198, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/bcf5949156f343f1ae4359ffe9cada4a, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/f39371934e4c446cbd3ce70fa5691aad] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp, totalSize=47.6 K 2024-11-27T07:24:37,804 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 840cf06b1fbf481d89b11637ec71caef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732692274471 2024-11-27T07:24:37,805 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting a00924dc2f034e94831bf9535d415198, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732692274592 2024-11-27T07:24:37,806 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting bcf5949156f343f1ae4359ffe9cada4a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732692275259 2024-11-27T07:24:37,807 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting f39371934e4c446cbd3ce70fa5691aad, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732692276566 2024-11-27T07:24:37,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741875_1051 (size=12151) 2024-11-27T07:24:37,822 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18c429ab939f89ba8ff6429b6bcb0851#C#compaction#39 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:24:37,823 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/d781860a378844fcaf536a58c8da5b72 is 50, key is test_row_0/C:col10/1732692276566/Put/seqid=0 2024-11-27T07:24:37,824 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/9c55ea41112440d9b90dca7ccd6c33fd 2024-11-27T07:24:37,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/b2e0de61f0ae430285ba091c4d65ad56 is 50, key is test_row_0/B:col10/1732692276793/Put/seqid=0 2024-11-27T07:24:37,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741877_1053 (size=12151) 2024-11-27T07:24:37,861 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/b2e0de61f0ae430285ba091c4d65ad56 2024-11-27T07:24:37,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741876_1052 (size=12561) 2024-11-27T07:24:37,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/4712da00af8942b5a80f288361bbcab4 is 50, key is test_row_0/C:col10/1732692276793/Put/seqid=0 2024-11-27T07:24:37,882 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/d781860a378844fcaf536a58c8da5b72 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/d781860a378844fcaf536a58c8da5b72 2024-11-27T07:24:37,893 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 18c429ab939f89ba8ff6429b6bcb0851/C of 18c429ab939f89ba8ff6429b6bcb0851 into d781860a378844fcaf536a58c8da5b72(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:24:37,893 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18c429ab939f89ba8ff6429b6bcb0851: 2024-11-27T07:24:37,893 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., storeName=18c429ab939f89ba8ff6429b6bcb0851/C, priority=12, startTime=1732692277727; duration=0sec 2024-11-27T07:24:37,893 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:24:37,893 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18c429ab939f89ba8ff6429b6bcb0851:C 2024-11-27T07:24:37,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741878_1054 (size=12151) 2024-11-27T07:24:37,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. as already flushing 2024-11-27T07:24:37,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 18c429ab939f89ba8ff6429b6bcb0851 2024-11-27T07:24:37,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-27T07:24:38,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:38,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50064 deadline: 1732692338062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:38,069 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:38,069 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:38,069 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:38,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:38,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50064 deadline: 1732692338188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:38,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:38,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:38,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:38,211 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/30df5ea5072946f4834f08170594efa6 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/30df5ea5072946f4834f08170594efa6 2024-11-27T07:24:38,231 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 18c429ab939f89ba8ff6429b6bcb0851/A of 18c429ab939f89ba8ff6429b6bcb0851 into 30df5ea5072946f4834f08170594efa6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:24:38,231 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18c429ab939f89ba8ff6429b6bcb0851: 2024-11-27T07:24:38,231 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., storeName=18c429ab939f89ba8ff6429b6bcb0851/A, priority=12, startTime=1732692277726; duration=0sec 2024-11-27T07:24:38,232 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:24:38,232 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18c429ab939f89ba8ff6429b6bcb0851:A 2024-11-27T07:24:38,322 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/4712da00af8942b5a80f288361bbcab4 2024-11-27T07:24:38,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/9c55ea41112440d9b90dca7ccd6c33fd as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/9c55ea41112440d9b90dca7ccd6c33fd 2024-11-27T07:24:38,346 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/9c55ea41112440d9b90dca7ccd6c33fd, entries=150, sequenceid=202, filesize=11.9 K 2024-11-27T07:24:38,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/b2e0de61f0ae430285ba091c4d65ad56 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/b2e0de61f0ae430285ba091c4d65ad56 2024-11-27T07:24:38,362 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/b2e0de61f0ae430285ba091c4d65ad56, entries=150, sequenceid=202, filesize=11.9 K 2024-11-27T07:24:38,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/4712da00af8942b5a80f288361bbcab4 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/4712da00af8942b5a80f288361bbcab4 2024-11-27T07:24:38,378 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/4712da00af8942b5a80f288361bbcab4, entries=150, sequenceid=202, filesize=11.9 K 2024-11-27T07:24:38,380 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for 18c429ab939f89ba8ff6429b6bcb0851 in 593ms, sequenceid=202, compaction requested=false 2024-11-27T07:24:38,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 18c429ab939f89ba8ff6429b6bcb0851: 2024-11-27T07:24:38,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:38,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-27T07:24:38,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-27T07:24:38,386 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-27T07:24:38,386 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0610 sec 2024-11-27T07:24:38,389 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.0710 sec 2024-11-27T07:24:38,402 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T07:24:38,402 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 18c429ab939f89ba8ff6429b6bcb0851 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-27T07:24:38,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 18c429ab939f89ba8ff6429b6bcb0851 2024-11-27T07:24:38,414 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/a7a35f378fb740f386214dea42606228 is 50, key is test_row_0/A:col10/1732692278044/Put/seqid=0 2024-11-27T07:24:38,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:38,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50064 deadline: 1732692338427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:38,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:38,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:38,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:38,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741879_1055 (size=14541) 2024-11-27T07:24:38,435 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/a7a35f378fb740f386214dea42606228 2024-11-27T07:24:38,450 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/73a04026511c4c4382082776dffe6c68 is 50, key is test_row_0/B:col10/1732692278044/Put/seqid=0 2024-11-27T07:24:38,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741880_1056 (size=12151) 2024-11-27T07:24:38,459 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/73a04026511c4c4382082776dffe6c68 2024-11-27T07:24:38,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-27T07:24:38,460 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T07:24:38,461 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T07:24:38,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-27T07:24:38,464 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T07:24:38,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-27T07:24:38,466 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T07:24:38,466 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T07:24:38,482 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/3428b337fc464c218dcf1e7542aefd71 is 50, key is test_row_0/C:col10/1732692278044/Put/seqid=0 2024-11-27T07:24:38,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741881_1057 (size=12151) 2024-11-27T07:24:38,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:38,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50064 deadline: 1732692338539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:38,543 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:38,543 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:38,543 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:38,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-27T07:24:38,619 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-27T07:24:38,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:38,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. as already flushing 2024-11-27T07:24:38,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:38,620 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:38,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:38,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:38,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:38,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50064 deadline: 1732692338749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:38,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:38,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:38,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:38,774 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-27T07:24:38,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:38,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. as already flushing 2024-11-27T07:24:38,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:38,775 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:38,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:38,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:38,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-27T07:24:38,894 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/3428b337fc464c218dcf1e7542aefd71 2024-11-27T07:24:38,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/a7a35f378fb740f386214dea42606228 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/a7a35f378fb740f386214dea42606228 2024-11-27T07:24:38,908 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/a7a35f378fb740f386214dea42606228, entries=200, sequenceid=233, filesize=14.2 K 2024-11-27T07:24:38,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/73a04026511c4c4382082776dffe6c68 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/73a04026511c4c4382082776dffe6c68 2024-11-27T07:24:38,916 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/73a04026511c4c4382082776dffe6c68, entries=150, sequenceid=233, filesize=11.9 K 2024-11-27T07:24:38,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/3428b337fc464c218dcf1e7542aefd71 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/3428b337fc464c218dcf1e7542aefd71 2024-11-27T07:24:38,925 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/3428b337fc464c218dcf1e7542aefd71, entries=150, sequenceid=233, filesize=11.9 K 2024-11-27T07:24:38,926 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 18c429ab939f89ba8ff6429b6bcb0851 in 524ms, sequenceid=233, compaction requested=true 2024-11-27T07:24:38,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 18c429ab939f89ba8ff6429b6bcb0851: 2024-11-27T07:24:38,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18c429ab939f89ba8ff6429b6bcb0851:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T07:24:38,927 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:24:38,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:24:38,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18c429ab939f89ba8ff6429b6bcb0851:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T07:24:38,927 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:24:38,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:24:38,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18c429ab939f89ba8ff6429b6bcb0851:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T07:24:38,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:24:38,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-27T07:24:38,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:38,929 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 18c429ab939f89ba8ff6429b6bcb0851 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-27T07:24:38,929 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39253 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:24:38,930 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1541): 18c429ab939f89ba8ff6429b6bcb0851/A is initiating minor compaction (all files) 2024-11-27T07:24:38,930 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18c429ab939f89ba8ff6429b6bcb0851/A in TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:38,930 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/30df5ea5072946f4834f08170594efa6, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/9c55ea41112440d9b90dca7ccd6c33fd, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/a7a35f378fb740f386214dea42606228] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp, totalSize=38.3 K 2024-11-27T07:24:38,931 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:24:38,931 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 18c429ab939f89ba8ff6429b6bcb0851/B is initiating minor compaction (all files) 2024-11-27T07:24:38,931 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18c429ab939f89ba8ff6429b6bcb0851/B in TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:38,931 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/8c049c9492a34f37bbee7dc9a00b9f78, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/b2e0de61f0ae430285ba091c4d65ad56, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/73a04026511c4c4382082776dffe6c68] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp, totalSize=36.0 K 2024-11-27T07:24:38,932 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c049c9492a34f37bbee7dc9a00b9f78, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732692276566 2024-11-27T07:24:38,933 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30df5ea5072946f4834f08170594efa6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732692276566 2024-11-27T07:24:38,933 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting b2e0de61f0ae430285ba091c4d65ad56, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1732692276754 2024-11-27T07:24:38,933 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c55ea41112440d9b90dca7ccd6c33fd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1732692276754 2024-11-27T07:24:38,934 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 73a04026511c4c4382082776dffe6c68, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732692278044 2024-11-27T07:24:38,934 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7a35f378fb740f386214dea42606228, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732692278044 2024-11-27T07:24:38,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/42bf3dfcfe0d4524b965446349e3a602 is 50, key is test_row_0/A:col10/1732692278424/Put/seqid=0 2024-11-27T07:24:38,958 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18c429ab939f89ba8ff6429b6bcb0851#B#compaction#46 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:24:38,958 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18c429ab939f89ba8ff6429b6bcb0851#A#compaction#47 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:24:38,959 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/7eeb849dd6394c42801a55a6a6d8c9c1 is 50, key is test_row_0/B:col10/1732692278044/Put/seqid=0 2024-11-27T07:24:38,959 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/a41b91706baa41e88d69da86feed98b8 is 50, key is test_row_0/A:col10/1732692278044/Put/seqid=0 2024-11-27T07:24:38,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741882_1058 (size=12151) 2024-11-27T07:24:38,983 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/42bf3dfcfe0d4524b965446349e3a602 2024-11-27T07:24:39,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/983434dc03bb4f7b99277779cc60705d is 50, key is test_row_0/B:col10/1732692278424/Put/seqid=0 2024-11-27T07:24:39,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741883_1059 (size=12663) 2024-11-27T07:24:39,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741884_1060 (size=12663) 2024-11-27T07:24:39,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741885_1061 (size=12151) 2024-11-27T07:24:39,035 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/983434dc03bb4f7b99277779cc60705d 2024-11-27T07:24:39,041 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/a41b91706baa41e88d69da86feed98b8 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/a41b91706baa41e88d69da86feed98b8 2024-11-27T07:24:39,048 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/7eeb849dd6394c42801a55a6a6d8c9c1 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/7eeb849dd6394c42801a55a6a6d8c9c1 2024-11-27T07:24:39,062 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 18c429ab939f89ba8ff6429b6bcb0851/A of 18c429ab939f89ba8ff6429b6bcb0851 into a41b91706baa41e88d69da86feed98b8(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:24:39,062 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 18c429ab939f89ba8ff6429b6bcb0851/B of 18c429ab939f89ba8ff6429b6bcb0851 into 7eeb849dd6394c42801a55a6a6d8c9c1(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:24:39,062 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18c429ab939f89ba8ff6429b6bcb0851: 2024-11-27T07:24:39,062 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18c429ab939f89ba8ff6429b6bcb0851: 2024-11-27T07:24:39,062 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., storeName=18c429ab939f89ba8ff6429b6bcb0851/A, priority=13, startTime=1732692278927; duration=0sec 2024-11-27T07:24:39,062 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., storeName=18c429ab939f89ba8ff6429b6bcb0851/B, priority=13, startTime=1732692278927; duration=0sec 2024-11-27T07:24:39,062 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:24:39,062 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18c429ab939f89ba8ff6429b6bcb0851:B 2024-11-27T07:24:39,063 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:24:39,064 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:24:39,064 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18c429ab939f89ba8ff6429b6bcb0851:A 2024-11-27T07:24:39,065 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:24:39,065 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 18c429ab939f89ba8ff6429b6bcb0851/C is initiating minor compaction (all files) 2024-11-27T07:24:39,065 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18c429ab939f89ba8ff6429b6bcb0851/C in TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:39,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/d756611ecf384003b98d02161399b974 is 50, key is test_row_0/C:col10/1732692278424/Put/seqid=0 2024-11-27T07:24:39,066 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/d781860a378844fcaf536a58c8da5b72, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/4712da00af8942b5a80f288361bbcab4, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/3428b337fc464c218dcf1e7542aefd71] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp, totalSize=36.0 K 2024-11-27T07:24:39,066 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting d781860a378844fcaf536a58c8da5b72, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732692276566 2024-11-27T07:24:39,067 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 4712da00af8942b5a80f288361bbcab4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1732692276754 2024-11-27T07:24:39,067 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 3428b337fc464c218dcf1e7542aefd71, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732692278044 2024-11-27T07:24:39,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 18c429ab939f89ba8ff6429b6bcb0851 2024-11-27T07:24:39,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. as already flushing 2024-11-27T07:24:39,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-27T07:24:39,101 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18c429ab939f89ba8ff6429b6bcb0851#C#compaction#50 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:24:39,102 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/145a808f2498471ca40626fc8b0da801 is 50, key is test_row_0/C:col10/1732692278044/Put/seqid=0 2024-11-27T07:24:39,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741886_1062 (size=12151) 2024-11-27T07:24:39,107 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/d756611ecf384003b98d02161399b974 2024-11-27T07:24:39,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/42bf3dfcfe0d4524b965446349e3a602 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/42bf3dfcfe0d4524b965446349e3a602 2024-11-27T07:24:39,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741887_1063 (size=12663) 2024-11-27T07:24:39,135 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/42bf3dfcfe0d4524b965446349e3a602, entries=150, sequenceid=241, filesize=11.9 K 2024-11-27T07:24:39,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/983434dc03bb4f7b99277779cc60705d as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/983434dc03bb4f7b99277779cc60705d 2024-11-27T07:24:39,140 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/145a808f2498471ca40626fc8b0da801 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/145a808f2498471ca40626fc8b0da801 2024-11-27T07:24:39,152 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/983434dc03bb4f7b99277779cc60705d, entries=150, sequenceid=241, filesize=11.9 K 2024-11-27T07:24:39,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/d756611ecf384003b98d02161399b974 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/d756611ecf384003b98d02161399b974 2024-11-27T07:24:39,155 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 18c429ab939f89ba8ff6429b6bcb0851/C of 18c429ab939f89ba8ff6429b6bcb0851 into 145a808f2498471ca40626fc8b0da801(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:24:39,155 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18c429ab939f89ba8ff6429b6bcb0851: 2024-11-27T07:24:39,155 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., storeName=18c429ab939f89ba8ff6429b6bcb0851/C, priority=13, startTime=1732692278927; duration=0sec 2024-11-27T07:24:39,155 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:24:39,155 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18c429ab939f89ba8ff6429b6bcb0851:C 2024-11-27T07:24:39,162 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/d756611ecf384003b98d02161399b974, entries=150, sequenceid=241, filesize=11.9 K 2024-11-27T07:24:39,164 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=120.76 KB/123660 for 18c429ab939f89ba8ff6429b6bcb0851 in 235ms, sequenceid=241, compaction requested=false 2024-11-27T07:24:39,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 18c429ab939f89ba8ff6429b6bcb0851: 2024-11-27T07:24:39,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:39,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-27T07:24:39,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-27T07:24:39,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 18c429ab939f89ba8ff6429b6bcb0851 2024-11-27T07:24:39,170 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T07:24:39,170 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 18c429ab939f89ba8ff6429b6bcb0851 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-27T07:24:39,171 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-27T07:24:39,172 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 700 msec 2024-11-27T07:24:39,175 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 711 msec 2024-11-27T07:24:39,186 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/f3d01c3e5df94102aa361838d6a85684 is 50, key is test_row_0/A:col10/1732692279169/Put/seqid=0 2024-11-27T07:24:39,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741888_1064 (size=17131) 2024-11-27T07:24:39,206 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/f3d01c3e5df94102aa361838d6a85684 2024-11-27T07:24:39,236 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/684329091a684328b3ced5c80038cd31 is 50, key is test_row_0/B:col10/1732692279169/Put/seqid=0 2024-11-27T07:24:39,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:39,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50064 deadline: 1732692339237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:39,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:39,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:39,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:39,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741889_1065 (size=12301) 2024-11-27T07:24:39,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:39,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:39,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50038 deadline: 1732692339349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:39,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50064 deadline: 1732692339349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:39,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:39,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:39,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:39,351 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 289954c9202d,42117,1732692259671 for 'test_row_1' in 18c429ab939f89ba8ff6429b6bcb0851 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T07:24:39,352 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:39,352 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:39,352 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:39,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:39,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50118 deadline: 1732692339359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:39,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:39,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50078 deadline: 1732692339359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:39,360 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 289954c9202d,42117,1732692259671 for 'test_row_2' in 18c429ab939f89ba8ff6429b6bcb0851 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8241 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T07:24:39,361 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:39,361 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:39,361 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:39,361 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 289954c9202d,42117,1732692259671 for 'test_row_1' in 18c429ab939f89ba8ff6429b6bcb0851 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8235 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T07:24:39,362 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:39,362 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:39,362 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:39,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:39,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50096 deadline: 1732692339369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:39,371 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 289954c9202d,42117,1732692259671 for 'test_row_0' in 18c429ab939f89ba8ff6429b6bcb0851 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8247 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T07:24:39,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:39,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:39,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:39,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:39,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50064 deadline: 1732692339559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:39,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:39,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:39,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:39,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-27T07:24:39,600 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T07:24:39,600 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T07:24:39,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-27T07:24:39,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-27T07:24:39,603 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T07:24:39,604 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T07:24:39,604 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T07:24:39,664 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/684329091a684328b3ced5c80038cd31 2024-11-27T07:24:39,684 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/53fbdd21b3ef4ca48bda4b6ac0a8d042 is 50, key is test_row_0/C:col10/1732692279169/Put/seqid=0 2024-11-27T07:24:39,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741890_1066 (size=12301) 2024-11-27T07:24:39,700 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/53fbdd21b3ef4ca48bda4b6ac0a8d042 2024-11-27T07:24:39,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-27T07:24:39,712 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/f3d01c3e5df94102aa361838d6a85684 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/f3d01c3e5df94102aa361838d6a85684 2024-11-27T07:24:39,719 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/f3d01c3e5df94102aa361838d6a85684, entries=250, sequenceid=266, filesize=16.7 K 2024-11-27T07:24:39,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/684329091a684328b3ced5c80038cd31 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/684329091a684328b3ced5c80038cd31 2024-11-27T07:24:39,730 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/684329091a684328b3ced5c80038cd31, entries=150, sequenceid=266, filesize=12.0 K 2024-11-27T07:24:39,732 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/53fbdd21b3ef4ca48bda4b6ac0a8d042 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/53fbdd21b3ef4ca48bda4b6ac0a8d042 2024-11-27T07:24:39,743 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/53fbdd21b3ef4ca48bda4b6ac0a8d042, entries=150, sequenceid=266, filesize=12.0 K 2024-11-27T07:24:39,744 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 18c429ab939f89ba8ff6429b6bcb0851 in 574ms, sequenceid=266, compaction requested=true 2024-11-27T07:24:39,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 18c429ab939f89ba8ff6429b6bcb0851: 2024-11-27T07:24:39,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18c429ab939f89ba8ff6429b6bcb0851:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T07:24:39,745 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:24:39,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:24:39,745 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:24:39,746 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:24:39,746 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 18c429ab939f89ba8ff6429b6bcb0851/B is initiating minor compaction (all files) 2024-11-27T07:24:39,747 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18c429ab939f89ba8ff6429b6bcb0851/B in TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:39,747 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/7eeb849dd6394c42801a55a6a6d8c9c1, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/983434dc03bb4f7b99277779cc60705d, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/684329091a684328b3ced5c80038cd31] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp, totalSize=36.2 K 2024-11-27T07:24:39,747 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41945 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:24:39,747 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 7eeb849dd6394c42801a55a6a6d8c9c1, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732692278044 2024-11-27T07:24:39,747 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1541): 18c429ab939f89ba8ff6429b6bcb0851/A is initiating minor compaction (all files) 2024-11-27T07:24:39,747 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18c429ab939f89ba8ff6429b6bcb0851/A in TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:39,748 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/a41b91706baa41e88d69da86feed98b8, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/42bf3dfcfe0d4524b965446349e3a602, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/f3d01c3e5df94102aa361838d6a85684] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp, totalSize=41.0 K 2024-11-27T07:24:39,748 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 983434dc03bb4f7b99277779cc60705d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732692278417 2024-11-27T07:24:39,748 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 684329091a684328b3ced5c80038cd31, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1732692279152 2024-11-27T07:24:39,748 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting a41b91706baa41e88d69da86feed98b8, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732692278044 2024-11-27T07:24:39,749 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42bf3dfcfe0d4524b965446349e3a602, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732692278417 2024-11-27T07:24:39,750 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3d01c3e5df94102aa361838d6a85684, keycount=250, bloomtype=ROW, size=16.7 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1732692279110 2024-11-27T07:24:39,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18c429ab939f89ba8ff6429b6bcb0851:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T07:24:39,758 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-27T07:24:39,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:39,759 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 18c429ab939f89ba8ff6429b6bcb0851 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-27T07:24:39,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:24:39,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18c429ab939f89ba8ff6429b6bcb0851:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T07:24:39,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:24:39,768 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18c429ab939f89ba8ff6429b6bcb0851#A#compaction#55 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:24:39,768 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/3ddbabbdc54b47548d1fa926743645e8 is 50, key is test_row_0/A:col10/1732692279169/Put/seqid=0 2024-11-27T07:24:39,769 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18c429ab939f89ba8ff6429b6bcb0851#B#compaction#54 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:24:39,769 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/11f78494130146bb86af212c6b85c0ea is 50, key is test_row_0/B:col10/1732692279169/Put/seqid=0 2024-11-27T07:24:39,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/a5351e8fbc9947cdae9a31b434b3b989 is 50, key is test_row_0/A:col10/1732692279234/Put/seqid=0 2024-11-27T07:24:39,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741891_1067 (size=12915) 2024-11-27T07:24:39,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741892_1068 (size=12915) 2024-11-27T07:24:39,791 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/11f78494130146bb86af212c6b85c0ea as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/11f78494130146bb86af212c6b85c0ea 2024-11-27T07:24:39,800 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/3ddbabbdc54b47548d1fa926743645e8 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/3ddbabbdc54b47548d1fa926743645e8 2024-11-27T07:24:39,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741893_1069 (size=12301) 2024-11-27T07:24:39,802 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/a5351e8fbc9947cdae9a31b434b3b989 2024-11-27T07:24:39,814 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 18c429ab939f89ba8ff6429b6bcb0851/B of 18c429ab939f89ba8ff6429b6bcb0851 into 11f78494130146bb86af212c6b85c0ea(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:24:39,814 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18c429ab939f89ba8ff6429b6bcb0851: 2024-11-27T07:24:39,814 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., storeName=18c429ab939f89ba8ff6429b6bcb0851/B, priority=13, startTime=1732692279745; duration=0sec 2024-11-27T07:24:39,815 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:24:39,815 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18c429ab939f89ba8ff6429b6bcb0851:B 2024-11-27T07:24:39,815 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:24:39,817 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:24:39,817 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 18c429ab939f89ba8ff6429b6bcb0851/C is initiating minor compaction (all files) 2024-11-27T07:24:39,818 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18c429ab939f89ba8ff6429b6bcb0851/C in TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:39,818 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/145a808f2498471ca40626fc8b0da801, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/d756611ecf384003b98d02161399b974, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/53fbdd21b3ef4ca48bda4b6ac0a8d042] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp, totalSize=36.2 K 2024-11-27T07:24:39,819 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 145a808f2498471ca40626fc8b0da801, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732692278044 2024-11-27T07:24:39,819 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting d756611ecf384003b98d02161399b974, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1732692278417 2024-11-27T07:24:39,821 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 18c429ab939f89ba8ff6429b6bcb0851/A of 18c429ab939f89ba8ff6429b6bcb0851 into 3ddbabbdc54b47548d1fa926743645e8(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:24:39,821 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18c429ab939f89ba8ff6429b6bcb0851: 2024-11-27T07:24:39,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/5aaa1143a1ad4050901f029d90685999 is 50, key is test_row_0/B:col10/1732692279234/Put/seqid=0 2024-11-27T07:24:39,821 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., storeName=18c429ab939f89ba8ff6429b6bcb0851/A, priority=13, startTime=1732692279745; duration=0sec 2024-11-27T07:24:39,821 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:24:39,821 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18c429ab939f89ba8ff6429b6bcb0851:A 2024-11-27T07:24:39,821 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 53fbdd21b3ef4ca48bda4b6ac0a8d042, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1732692279152 2024-11-27T07:24:39,845 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18c429ab939f89ba8ff6429b6bcb0851#C#compaction#58 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:24:39,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741894_1070 (size=12301) 2024-11-27T07:24:39,846 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/dd6b31f4fada4d02a5a7b2960bf45cf8 is 50, key is test_row_0/C:col10/1732692279169/Put/seqid=0 2024-11-27T07:24:39,846 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/5aaa1143a1ad4050901f029d90685999 2024-11-27T07:24:39,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/0c3e5d08e1a74ea6bfdedbfbafeb665a is 50, key is test_row_0/C:col10/1732692279234/Put/seqid=0 2024-11-27T07:24:39,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. as already flushing 2024-11-27T07:24:39,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 18c429ab939f89ba8ff6429b6bcb0851 2024-11-27T07:24:39,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741895_1071 (size=12915) 2024-11-27T07:24:39,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741896_1072 (size=12301) 2024-11-27T07:24:39,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-27T07:24:39,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:39,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50064 deadline: 1732692339943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:39,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:39,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:39,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:40,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:40,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50064 deadline: 1732692340049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:40,052 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:40,052 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:40,052 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:40,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-27T07:24:40,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:40,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50064 deadline: 1732692340259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:40,263 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:40,263 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:40,263 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:40,288 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/dd6b31f4fada4d02a5a7b2960bf45cf8 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/dd6b31f4fada4d02a5a7b2960bf45cf8 2024-11-27T07:24:40,289 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/0c3e5d08e1a74ea6bfdedbfbafeb665a 2024-11-27T07:24:40,302 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 18c429ab939f89ba8ff6429b6bcb0851/C of 18c429ab939f89ba8ff6429b6bcb0851 into dd6b31f4fada4d02a5a7b2960bf45cf8(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:24:40,302 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18c429ab939f89ba8ff6429b6bcb0851: 2024-11-27T07:24:40,302 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., storeName=18c429ab939f89ba8ff6429b6bcb0851/C, priority=13, startTime=1732692279761; duration=0sec 2024-11-27T07:24:40,302 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:24:40,302 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18c429ab939f89ba8ff6429b6bcb0851:C 2024-11-27T07:24:40,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/a5351e8fbc9947cdae9a31b434b3b989 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/a5351e8fbc9947cdae9a31b434b3b989 2024-11-27T07:24:40,309 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/a5351e8fbc9947cdae9a31b434b3b989, entries=150, sequenceid=280, filesize=12.0 K 2024-11-27T07:24:40,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/5aaa1143a1ad4050901f029d90685999 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/5aaa1143a1ad4050901f029d90685999 2024-11-27T07:24:40,316 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/5aaa1143a1ad4050901f029d90685999, entries=150, sequenceid=280, filesize=12.0 K 2024-11-27T07:24:40,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/0c3e5d08e1a74ea6bfdedbfbafeb665a as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/0c3e5d08e1a74ea6bfdedbfbafeb665a 2024-11-27T07:24:40,324 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/0c3e5d08e1a74ea6bfdedbfbafeb665a, entries=150, sequenceid=280, filesize=12.0 K 2024-11-27T07:24:40,325 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 18c429ab939f89ba8ff6429b6bcb0851 in 567ms, sequenceid=280, compaction requested=false 2024-11-27T07:24:40,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 18c429ab939f89ba8ff6429b6bcb0851: 2024-11-27T07:24:40,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:40,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-27T07:24:40,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-27T07:24:40,329 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-27T07:24:40,329 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 723 msec 2024-11-27T07:24:40,331 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 729 msec 2024-11-27T07:24:40,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 18c429ab939f89ba8ff6429b6bcb0851 2024-11-27T07:24:40,571 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T07:24:40,571 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 18c429ab939f89ba8ff6429b6bcb0851 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-27T07:24:40,578 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/61a5df9c28c043ba9aa8443d06363d12 is 50, key is test_row_0/A:col10/1732692279941/Put/seqid=0 2024-11-27T07:24:40,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741897_1073 (size=14741) 2024-11-27T07:24:40,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:40,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50064 deadline: 1732692340605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:40,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:40,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:40,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:40,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:40,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50064 deadline: 1732692340719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:40,720 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:40,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:40,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:40,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-27T07:24:40,740 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T07:24:40,740 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T07:24:40,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-27T07:24:40,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-27T07:24:40,742 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T07:24:40,743 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T07:24:40,743 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T07:24:40,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-27T07:24:40,896 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-27T07:24:40,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:40,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. as already flushing 2024-11-27T07:24:40,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:40,896 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:40,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:40,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:40,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:40,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50064 deadline: 1732692340929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:40,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:40,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:40,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:40,992 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/61a5df9c28c043ba9aa8443d06363d12 2024-11-27T07:24:41,002 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/baae975a28094037ba4cd653541b4797 is 50, key is test_row_0/B:col10/1732692279941/Put/seqid=0 2024-11-27T07:24:41,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741898_1074 (size=12301) 2024-11-27T07:24:41,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-27T07:24:41,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:41,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. as already flushing 2024-11-27T07:24:41,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:41,050 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:41,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:41,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:41,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-27T07:24:41,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-27T07:24:41,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:41,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. as already flushing 2024-11-27T07:24:41,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:41,204 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:41,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:41,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:41,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:41,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50064 deadline: 1732692341239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:41,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:41,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:41,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:41,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-27T07:24:41,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:41,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. as already flushing 2024-11-27T07:24:41,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:41,357 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:41,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:41,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:41,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-27T07:24:41,409 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/baae975a28094037ba4cd653541b4797 2024-11-27T07:24:41,417 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/9e9d7bd4556a4d6ba9c4b31fc1296a4d is 50, key is test_row_0/C:col10/1732692279941/Put/seqid=0 2024-11-27T07:24:41,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741899_1075 (size=12301) 2024-11-27T07:24:41,430 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/9e9d7bd4556a4d6ba9c4b31fc1296a4d 2024-11-27T07:24:41,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/61a5df9c28c043ba9aa8443d06363d12 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/61a5df9c28c043ba9aa8443d06363d12 2024-11-27T07:24:41,442 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/61a5df9c28c043ba9aa8443d06363d12, entries=200, sequenceid=306, filesize=14.4 K 2024-11-27T07:24:41,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/baae975a28094037ba4cd653541b4797 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/baae975a28094037ba4cd653541b4797 2024-11-27T07:24:41,450 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/baae975a28094037ba4cd653541b4797, entries=150, sequenceid=306, filesize=12.0 K 2024-11-27T07:24:41,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/9e9d7bd4556a4d6ba9c4b31fc1296a4d as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/9e9d7bd4556a4d6ba9c4b31fc1296a4d 2024-11-27T07:24:41,457 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/9e9d7bd4556a4d6ba9c4b31fc1296a4d, entries=150, sequenceid=306, filesize=12.0 K 2024-11-27T07:24:41,458 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 18c429ab939f89ba8ff6429b6bcb0851 in 887ms, sequenceid=306, compaction requested=true 2024-11-27T07:24:41,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 18c429ab939f89ba8ff6429b6bcb0851: 2024-11-27T07:24:41,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18c429ab939f89ba8ff6429b6bcb0851:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T07:24:41,459 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:24:41,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:24:41,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18c429ab939f89ba8ff6429b6bcb0851:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T07:24:41,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:24:41,459 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:24:41,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18c429ab939f89ba8ff6429b6bcb0851:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T07:24:41,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:24:41,460 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39957 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:24:41,460 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:24:41,460 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1541): 18c429ab939f89ba8ff6429b6bcb0851/A is initiating minor compaction (all files) 2024-11-27T07:24:41,460 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 18c429ab939f89ba8ff6429b6bcb0851/B is initiating minor compaction (all files) 2024-11-27T07:24:41,460 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18c429ab939f89ba8ff6429b6bcb0851/A in TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:41,460 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18c429ab939f89ba8ff6429b6bcb0851/B in TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:41,460 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/3ddbabbdc54b47548d1fa926743645e8, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/a5351e8fbc9947cdae9a31b434b3b989, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/61a5df9c28c043ba9aa8443d06363d12] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp, totalSize=39.0 K 2024-11-27T07:24:41,460 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/11f78494130146bb86af212c6b85c0ea, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/5aaa1143a1ad4050901f029d90685999, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/baae975a28094037ba4cd653541b4797] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp, totalSize=36.6 K 2024-11-27T07:24:41,461 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ddbabbdc54b47548d1fa926743645e8, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1732692279152 2024-11-27T07:24:41,461 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 11f78494130146bb86af212c6b85c0ea, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1732692279152 2024-11-27T07:24:41,464 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 5aaa1143a1ad4050901f029d90685999, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1732692279217 2024-11-27T07:24:41,464 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5351e8fbc9947cdae9a31b434b3b989, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1732692279217 2024-11-27T07:24:41,464 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting baae975a28094037ba4cd653541b4797, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1732692279929 2024-11-27T07:24:41,465 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61a5df9c28c043ba9aa8443d06363d12, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1732692279929 2024-11-27T07:24:41,482 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18c429ab939f89ba8ff6429b6bcb0851#B#compaction#63 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:24:41,483 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/4f80d7236c5d466a8b77b190717860e7 is 50, key is test_row_0/B:col10/1732692279941/Put/seqid=0 2024-11-27T07:24:41,491 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18c429ab939f89ba8ff6429b6bcb0851#A#compaction#64 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:24:41,492 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/9f4861af7ce048f3ac81ccbfebb0bf1a is 50, key is test_row_0/A:col10/1732692279941/Put/seqid=0 2024-11-27T07:24:41,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741900_1076 (size=13017) 2024-11-27T07:24:41,510 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-27T07:24:41,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:41,511 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 18c429ab939f89ba8ff6429b6bcb0851 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-27T07:24:41,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/d6cdd47e7f344a3b81ed9986042487e8 is 50, key is test_row_0/A:col10/1732692280603/Put/seqid=0 2024-11-27T07:24:41,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741901_1077 (size=13017) 2024-11-27T07:24:41,523 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/4f80d7236c5d466a8b77b190717860e7 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/4f80d7236c5d466a8b77b190717860e7 2024-11-27T07:24:41,533 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/9f4861af7ce048f3ac81ccbfebb0bf1a as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/9f4861af7ce048f3ac81ccbfebb0bf1a 2024-11-27T07:24:41,535 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 18c429ab939f89ba8ff6429b6bcb0851/B of 18c429ab939f89ba8ff6429b6bcb0851 into 4f80d7236c5d466a8b77b190717860e7(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:24:41,535 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18c429ab939f89ba8ff6429b6bcb0851: 2024-11-27T07:24:41,535 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., storeName=18c429ab939f89ba8ff6429b6bcb0851/B, priority=13, startTime=1732692281459; duration=0sec 2024-11-27T07:24:41,535 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:24:41,535 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18c429ab939f89ba8ff6429b6bcb0851:B 2024-11-27T07:24:41,535 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:24:41,540 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:24:41,541 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 18c429ab939f89ba8ff6429b6bcb0851/C is initiating minor compaction (all files) 2024-11-27T07:24:41,541 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18c429ab939f89ba8ff6429b6bcb0851/C in TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:41,541 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/dd6b31f4fada4d02a5a7b2960bf45cf8, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/0c3e5d08e1a74ea6bfdedbfbafeb665a, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/9e9d7bd4556a4d6ba9c4b31fc1296a4d] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp, totalSize=36.6 K 2024-11-27T07:24:41,543 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting dd6b31f4fada4d02a5a7b2960bf45cf8, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1732692279152 2024-11-27T07:24:41,546 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c3e5d08e1a74ea6bfdedbfbafeb665a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1732692279217 2024-11-27T07:24:41,546 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 18c429ab939f89ba8ff6429b6bcb0851/A of 18c429ab939f89ba8ff6429b6bcb0851 into 9f4861af7ce048f3ac81ccbfebb0bf1a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:24:41,546 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18c429ab939f89ba8ff6429b6bcb0851: 2024-11-27T07:24:41,546 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., storeName=18c429ab939f89ba8ff6429b6bcb0851/A, priority=13, startTime=1732692281459; duration=0sec 2024-11-27T07:24:41,546 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:24:41,546 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18c429ab939f89ba8ff6429b6bcb0851:A 2024-11-27T07:24:41,547 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e9d7bd4556a4d6ba9c4b31fc1296a4d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1732692279929 2024-11-27T07:24:41,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741902_1078 (size=12301) 2024-11-27T07:24:41,561 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/d6cdd47e7f344a3b81ed9986042487e8 2024-11-27T07:24:41,563 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18c429ab939f89ba8ff6429b6bcb0851#C#compaction#66 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:24:41,564 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/381b4e0fca1544a5b9a36b7d17b77b73 is 50, key is test_row_0/C:col10/1732692279941/Put/seqid=0 2024-11-27T07:24:41,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/a0c66d9c89a243d4921a063143d0c5ea is 50, key is test_row_0/B:col10/1732692280603/Put/seqid=0 2024-11-27T07:24:41,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741903_1079 (size=13017) 2024-11-27T07:24:41,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741904_1080 (size=12301) 2024-11-27T07:24:41,593 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/381b4e0fca1544a5b9a36b7d17b77b73 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/381b4e0fca1544a5b9a36b7d17b77b73 2024-11-27T07:24:41,600 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 18c429ab939f89ba8ff6429b6bcb0851/C of 18c429ab939f89ba8ff6429b6bcb0851 into 381b4e0fca1544a5b9a36b7d17b77b73(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:24:41,600 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18c429ab939f89ba8ff6429b6bcb0851: 2024-11-27T07:24:41,601 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., storeName=18c429ab939f89ba8ff6429b6bcb0851/C, priority=13, startTime=1732692281459; duration=0sec 2024-11-27T07:24:41,601 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:24:41,601 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18c429ab939f89ba8ff6429b6bcb0851:C 2024-11-27T07:24:41,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 18c429ab939f89ba8ff6429b6bcb0851 2024-11-27T07:24:41,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. as already flushing 2024-11-27T07:24:41,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:41,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50064 deadline: 1732692341841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:41,845 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:41,845 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:41,845 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:41,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-27T07:24:41,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:41,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50064 deadline: 1732692341949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:41,950 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:41,950 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:41,950 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:41,986 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/a0c66d9c89a243d4921a063143d0c5ea 2024-11-27T07:24:42,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/8a3728f7820f4fc5ac64f985496de9c5 is 50, key is test_row_0/C:col10/1732692280603/Put/seqid=0 2024-11-27T07:24:42,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741905_1081 (size=12301) 2024-11-27T07:24:42,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:42,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50064 deadline: 1732692342159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:42,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:42,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:42,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:42,427 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/8a3728f7820f4fc5ac64f985496de9c5 2024-11-27T07:24:42,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:42,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 244 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50064 deadline: 1732692342469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:42,470 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:42,470 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:42,470 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:42,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/d6cdd47e7f344a3b81ed9986042487e8 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/d6cdd47e7f344a3b81ed9986042487e8 2024-11-27T07:24:42,498 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/d6cdd47e7f344a3b81ed9986042487e8, entries=150, sequenceid=319, filesize=12.0 K 2024-11-27T07:24:42,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/a0c66d9c89a243d4921a063143d0c5ea as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/a0c66d9c89a243d4921a063143d0c5ea 2024-11-27T07:24:42,507 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/a0c66d9c89a243d4921a063143d0c5ea, entries=150, sequenceid=319, filesize=12.0 K 2024-11-27T07:24:42,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/8a3728f7820f4fc5ac64f985496de9c5 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/8a3728f7820f4fc5ac64f985496de9c5 2024-11-27T07:24:42,516 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/8a3728f7820f4fc5ac64f985496de9c5, entries=150, sequenceid=319, filesize=12.0 K 2024-11-27T07:24:42,517 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 18c429ab939f89ba8ff6429b6bcb0851 in 1007ms, sequenceid=319, compaction requested=false 2024-11-27T07:24:42,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 18c429ab939f89ba8ff6429b6bcb0851: 2024-11-27T07:24:42,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:42,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-27T07:24:42,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-27T07:24:42,521 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-27T07:24:42,521 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7760 sec 2024-11-27T07:24:42,523 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.7820 sec 2024-11-27T07:24:42,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-27T07:24:42,889 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T07:24:42,889 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T07:24:42,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-27T07:24:42,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-27T07:24:42,891 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T07:24:42,892 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T07:24:42,892 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T07:24:42,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 18c429ab939f89ba8ff6429b6bcb0851 2024-11-27T07:24:42,982 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T07:24:42,982 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 18c429ab939f89ba8ff6429b6bcb0851 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-27T07:24:42,990 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/fb6240e37c014f099e895c3915c52ae5 is 50, key is test_row_0/A:col10/1732692281829/Put/seqid=0 2024-11-27T07:24:42,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-27T07:24:43,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741906_1082 (size=12301) 2024-11-27T07:24:43,044 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-27T07:24:43,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:43,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. as already flushing 2024-11-27T07:24:43,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:43,045 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:43,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:43,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:43,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:43,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 255 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50064 deadline: 1732692343047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:43,055 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:43,055 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:43,055 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:43,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:43,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 256 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50064 deadline: 1732692343159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:43,160 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:43,160 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:43,160 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:43,197 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-27T07:24:43,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:43,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. as already flushing 2024-11-27T07:24:43,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:43,199 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:43,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:43,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:43,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-27T07:24:43,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-27T07:24:43,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:43,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. as already flushing 2024-11-27T07:24:43,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:43,352 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:43,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:43,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:43,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:43,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 257 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50064 deadline: 1732692343369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:43,370 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:43,370 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:43,370 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:43,411 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/fb6240e37c014f099e895c3915c52ae5 2024-11-27T07:24:43,425 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/70d05ab955cf4ea19384397a656400e5 is 50, key is test_row_0/B:col10/1732692281829/Put/seqid=0 2024-11-27T07:24:43,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741907_1083 (size=12301) 2024-11-27T07:24:43,434 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/70d05ab955cf4ea19384397a656400e5 2024-11-27T07:24:43,448 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/314306bc145a4f4da646de9e73abd915 is 50, key is test_row_0/C:col10/1732692281829/Put/seqid=0 2024-11-27T07:24:43,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741908_1084 (size=12301) 2024-11-27T07:24:43,505 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-27T07:24:43,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:43,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. as already flushing 2024-11-27T07:24:43,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:43,506 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:43,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:43,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:43,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-27T07:24:43,660 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-27T07:24:43,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:43,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. as already flushing 2024-11-27T07:24:43,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:43,660 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:43,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:43,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:43,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:43,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 258 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50064 deadline: 1732692343679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:43,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:43,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:43,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:43,814 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-27T07:24:43,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:43,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. as already flushing 2024-11-27T07:24:43,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:43,814 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:43,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:43,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:43,870 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/314306bc145a4f4da646de9e73abd915 2024-11-27T07:24:43,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/fb6240e37c014f099e895c3915c52ae5 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/fb6240e37c014f099e895c3915c52ae5 2024-11-27T07:24:43,889 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/fb6240e37c014f099e895c3915c52ae5, entries=150, sequenceid=346, filesize=12.0 K 2024-11-27T07:24:43,891 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/70d05ab955cf4ea19384397a656400e5 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/70d05ab955cf4ea19384397a656400e5 2024-11-27T07:24:43,899 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/70d05ab955cf4ea19384397a656400e5, entries=150, sequenceid=346, filesize=12.0 K 2024-11-27T07:24:43,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/314306bc145a4f4da646de9e73abd915 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/314306bc145a4f4da646de9e73abd915 2024-11-27T07:24:43,911 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/314306bc145a4f4da646de9e73abd915, entries=150, sequenceid=346, filesize=12.0 K 2024-11-27T07:24:43,913 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 18c429ab939f89ba8ff6429b6bcb0851 in 930ms, sequenceid=346, compaction requested=true 2024-11-27T07:24:43,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 18c429ab939f89ba8ff6429b6bcb0851: 2024-11-27T07:24:43,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18c429ab939f89ba8ff6429b6bcb0851:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T07:24:43,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:24:43,913 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:24:43,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18c429ab939f89ba8ff6429b6bcb0851:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T07:24:43,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:24:43,913 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:24:43,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18c429ab939f89ba8ff6429b6bcb0851:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T07:24:43,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:24:43,914 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:24:43,915 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 18c429ab939f89ba8ff6429b6bcb0851/B is initiating minor compaction (all files) 2024-11-27T07:24:43,915 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18c429ab939f89ba8ff6429b6bcb0851/B in TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:43,915 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/4f80d7236c5d466a8b77b190717860e7, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/a0c66d9c89a243d4921a063143d0c5ea, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/70d05ab955cf4ea19384397a656400e5] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp, totalSize=36.7 K 2024-11-27T07:24:43,915 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:24:43,915 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1541): 18c429ab939f89ba8ff6429b6bcb0851/A is initiating minor compaction (all files) 2024-11-27T07:24:43,915 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18c429ab939f89ba8ff6429b6bcb0851/A in TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:43,916 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/9f4861af7ce048f3ac81ccbfebb0bf1a, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/d6cdd47e7f344a3b81ed9986042487e8, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/fb6240e37c014f099e895c3915c52ae5] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp, totalSize=36.7 K 2024-11-27T07:24:43,916 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f80d7236c5d466a8b77b190717860e7, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1732692279929 2024-11-27T07:24:43,916 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f4861af7ce048f3ac81ccbfebb0bf1a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1732692279929 2024-11-27T07:24:43,917 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6cdd47e7f344a3b81ed9986042487e8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1732692280593 2024-11-27T07:24:43,917 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting a0c66d9c89a243d4921a063143d0c5ea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1732692280593 2024-11-27T07:24:43,918 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 70d05ab955cf4ea19384397a656400e5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1732692281829 2024-11-27T07:24:43,918 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb6240e37c014f099e895c3915c52ae5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1732692281829 2024-11-27T07:24:43,929 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18c429ab939f89ba8ff6429b6bcb0851#A#compaction#72 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:24:43,929 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/6d7d250887d54274929c21d5eaae0ba9 is 50, key is test_row_0/A:col10/1732692281829/Put/seqid=0 2024-11-27T07:24:43,930 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18c429ab939f89ba8ff6429b6bcb0851#B#compaction#73 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:24:43,931 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/8016129177b046a299835e2a81f16168 is 50, key is test_row_0/B:col10/1732692281829/Put/seqid=0 2024-11-27T07:24:43,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741909_1085 (size=13119) 2024-11-27T07:24:43,947 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/6d7d250887d54274929c21d5eaae0ba9 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/6d7d250887d54274929c21d5eaae0ba9 2024-11-27T07:24:43,956 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 18c429ab939f89ba8ff6429b6bcb0851/A of 18c429ab939f89ba8ff6429b6bcb0851 into 6d7d250887d54274929c21d5eaae0ba9(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:24:43,956 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18c429ab939f89ba8ff6429b6bcb0851: 2024-11-27T07:24:43,957 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., storeName=18c429ab939f89ba8ff6429b6bcb0851/A, priority=13, startTime=1732692283913; duration=0sec 2024-11-27T07:24:43,957 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:24:43,957 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18c429ab939f89ba8ff6429b6bcb0851:A 2024-11-27T07:24:43,957 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:24:43,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741910_1086 (size=13119) 2024-11-27T07:24:43,959 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:24:43,959 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1541): 18c429ab939f89ba8ff6429b6bcb0851/C is initiating minor compaction (all files) 2024-11-27T07:24:43,959 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18c429ab939f89ba8ff6429b6bcb0851/C in TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:43,959 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/381b4e0fca1544a5b9a36b7d17b77b73, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/8a3728f7820f4fc5ac64f985496de9c5, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/314306bc145a4f4da646de9e73abd915] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp, totalSize=36.7 K 2024-11-27T07:24:43,960 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 381b4e0fca1544a5b9a36b7d17b77b73, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1732692279929 2024-11-27T07:24:43,961 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a3728f7820f4fc5ac64f985496de9c5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1732692280593 2024-11-27T07:24:43,963 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 314306bc145a4f4da646de9e73abd915, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1732692281829 2024-11-27T07:24:43,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-27T07:24:43,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:43,969 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing 18c429ab939f89ba8ff6429b6bcb0851 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-27T07:24:43,978 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18c429ab939f89ba8ff6429b6bcb0851#C#compaction#74 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:24:43,979 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/483df80637d045e88c6f1b5afc01ad31 is 50, key is test_row_0/C:col10/1732692281829/Put/seqid=0 2024-11-27T07:24:43,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/d656e0c13e494a699c4a3b969318f612 is 50, key is test_row_0/A:col10/1732692283002/Put/seqid=0 2024-11-27T07:24:44,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741912_1088 (size=12301) 2024-11-27T07:24:44,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741911_1087 (size=13119) 2024-11-27T07:24:44,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-27T07:24:44,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. as already flushing 2024-11-27T07:24:44,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 18c429ab939f89ba8ff6429b6bcb0851 2024-11-27T07:24:44,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:44,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 280 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50064 deadline: 1732692344286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:44,288 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:44,288 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:44,288 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:44,372 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/8016129177b046a299835e2a81f16168 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/8016129177b046a299835e2a81f16168 2024-11-27T07:24:44,381 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 18c429ab939f89ba8ff6429b6bcb0851/B of 18c429ab939f89ba8ff6429b6bcb0851 into 8016129177b046a299835e2a81f16168(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:24:44,381 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18c429ab939f89ba8ff6429b6bcb0851: 2024-11-27T07:24:44,381 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., storeName=18c429ab939f89ba8ff6429b6bcb0851/B, priority=13, startTime=1732692283913; duration=0sec 2024-11-27T07:24:44,382 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:24:44,382 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18c429ab939f89ba8ff6429b6bcb0851:B 2024-11-27T07:24:44,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:44,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 281 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50064 deadline: 1732692344400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:44,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:44,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:44,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:44,419 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/d656e0c13e494a699c4a3b969318f612 2024-11-27T07:24:44,437 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/483df80637d045e88c6f1b5afc01ad31 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/483df80637d045e88c6f1b5afc01ad31 2024-11-27T07:24:44,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/c28d852f7f034c34a185e80d5caca9c5 is 50, key is test_row_0/B:col10/1732692283002/Put/seqid=0 2024-11-27T07:24:44,454 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 18c429ab939f89ba8ff6429b6bcb0851/C of 18c429ab939f89ba8ff6429b6bcb0851 into 483df80637d045e88c6f1b5afc01ad31(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:24:44,454 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18c429ab939f89ba8ff6429b6bcb0851: 2024-11-27T07:24:44,454 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., storeName=18c429ab939f89ba8ff6429b6bcb0851/C, priority=13, startTime=1732692283913; duration=0sec 2024-11-27T07:24:44,454 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:24:44,454 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18c429ab939f89ba8ff6429b6bcb0851:C 2024-11-27T07:24:44,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741913_1089 (size=12301) 2024-11-27T07:24:44,458 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/c28d852f7f034c34a185e80d5caca9c5 2024-11-27T07:24:44,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/15da2e53db0141e99d458cc5573156ab is 50, key is test_row_0/C:col10/1732692283002/Put/seqid=0 2024-11-27T07:24:44,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741914_1090 (size=12301) 2024-11-27T07:24:44,484 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/15da2e53db0141e99d458cc5573156ab 2024-11-27T07:24:44,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/d656e0c13e494a699c4a3b969318f612 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/d656e0c13e494a699c4a3b969318f612 2024-11-27T07:24:44,502 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/d656e0c13e494a699c4a3b969318f612, entries=150, sequenceid=359, filesize=12.0 K 2024-11-27T07:24:44,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/c28d852f7f034c34a185e80d5caca9c5 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/c28d852f7f034c34a185e80d5caca9c5 2024-11-27T07:24:44,515 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/c28d852f7f034c34a185e80d5caca9c5, entries=150, sequenceid=359, filesize=12.0 K 2024-11-27T07:24:44,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/15da2e53db0141e99d458cc5573156ab as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/15da2e53db0141e99d458cc5573156ab 2024-11-27T07:24:44,527 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/15da2e53db0141e99d458cc5573156ab, entries=150, sequenceid=359, filesize=12.0 K 2024-11-27T07:24:44,529 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 18c429ab939f89ba8ff6429b6bcb0851 in 559ms, sequenceid=359, compaction requested=false 2024-11-27T07:24:44,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 18c429ab939f89ba8ff6429b6bcb0851: 2024-11-27T07:24:44,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:44,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-27T07:24:44,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-27T07:24:44,534 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-27T07:24:44,534 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6390 sec 2024-11-27T07:24:44,536 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 1.6440 sec 2024-11-27T07:24:44,615 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T07:24:44,615 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 18c429ab939f89ba8ff6429b6bcb0851 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-27T07:24:44,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 18c429ab939f89ba8ff6429b6bcb0851 2024-11-27T07:24:44,621 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/d210d4bdebf9444ab914cda2d38b3adc is 50, key is test_row_0/A:col10/1732692284274/Put/seqid=0 2024-11-27T07:24:44,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741915_1091 (size=12301) 2024-11-27T07:24:44,631 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/d210d4bdebf9444ab914cda2d38b3adc 2024-11-27T07:24:44,643 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/0f565176e5804256b5a70623de2faa12 is 50, key is test_row_0/B:col10/1732692284274/Put/seqid=0 2024-11-27T07:24:44,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741916_1092 (size=12301) 2024-11-27T07:24:44,670 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/0f565176e5804256b5a70623de2faa12 2024-11-27T07:24:44,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:44,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 291 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50064 deadline: 1732692344684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:44,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:44,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:44,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:44,694 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/262276ecf66b47928edb55c47ab467e1 is 50, key is test_row_0/C:col10/1732692284274/Put/seqid=0 2024-11-27T07:24:44,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741917_1093 (size=12301) 2024-11-27T07:24:44,721 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/262276ecf66b47928edb55c47ab467e1 2024-11-27T07:24:44,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/d210d4bdebf9444ab914cda2d38b3adc as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/d210d4bdebf9444ab914cda2d38b3adc 2024-11-27T07:24:44,744 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/d210d4bdebf9444ab914cda2d38b3adc, entries=150, sequenceid=386, filesize=12.0 K 2024-11-27T07:24:44,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/0f565176e5804256b5a70623de2faa12 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/0f565176e5804256b5a70623de2faa12 2024-11-27T07:24:44,756 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/0f565176e5804256b5a70623de2faa12, entries=150, sequenceid=386, filesize=12.0 K 2024-11-27T07:24:44,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/262276ecf66b47928edb55c47ab467e1 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/262276ecf66b47928edb55c47ab467e1 2024-11-27T07:24:44,770 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/262276ecf66b47928edb55c47ab467e1, entries=150, sequenceid=386, filesize=12.0 K 2024-11-27T07:24:44,771 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 18c429ab939f89ba8ff6429b6bcb0851 in 156ms, sequenceid=386, compaction requested=true 2024-11-27T07:24:44,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 18c429ab939f89ba8ff6429b6bcb0851: 2024-11-27T07:24:44,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18c429ab939f89ba8ff6429b6bcb0851:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T07:24:44,773 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:24:44,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:24:44,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18c429ab939f89ba8ff6429b6bcb0851:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T07:24:44,773 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:24:44,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:24:44,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18c429ab939f89ba8ff6429b6bcb0851:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T07:24:44,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:24:44,775 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:24:44,775 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1541): 18c429ab939f89ba8ff6429b6bcb0851/A is initiating minor compaction (all files) 2024-11-27T07:24:44,775 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18c429ab939f89ba8ff6429b6bcb0851/A in TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:44,775 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/6d7d250887d54274929c21d5eaae0ba9, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/d656e0c13e494a699c4a3b969318f612, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/d210d4bdebf9444ab914cda2d38b3adc] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp, totalSize=36.8 K 2024-11-27T07:24:44,776 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:24:44,776 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 18c429ab939f89ba8ff6429b6bcb0851/B is initiating minor compaction (all files) 2024-11-27T07:24:44,776 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18c429ab939f89ba8ff6429b6bcb0851/B in TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:44,776 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/8016129177b046a299835e2a81f16168, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/c28d852f7f034c34a185e80d5caca9c5, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/0f565176e5804256b5a70623de2faa12] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp, totalSize=36.8 K 2024-11-27T07:24:44,776 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d7d250887d54274929c21d5eaae0ba9, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1732692281829 2024-11-27T07:24:44,776 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 8016129177b046a299835e2a81f16168, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1732692281829 2024-11-27T07:24:44,777 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting d656e0c13e494a699c4a3b969318f612, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1732692283002 2024-11-27T07:24:44,777 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting c28d852f7f034c34a185e80d5caca9c5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1732692283002 2024-11-27T07:24:44,778 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f565176e5804256b5a70623de2faa12, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1732692284240 2024-11-27T07:24:44,779 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting d210d4bdebf9444ab914cda2d38b3adc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1732692284240 2024-11-27T07:24:44,805 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18c429ab939f89ba8ff6429b6bcb0851#B#compaction#81 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:24:44,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 18c429ab939f89ba8ff6429b6bcb0851 2024-11-27T07:24:44,808 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T07:24:44,808 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 18c429ab939f89ba8ff6429b6bcb0851 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-27T07:24:44,809 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/4f4f68ab2be840139abacf50c4eec012 is 50, key is test_row_0/B:col10/1732692284274/Put/seqid=0 2024-11-27T07:24:44,815 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18c429ab939f89ba8ff6429b6bcb0851#A#compaction#82 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:24:44,816 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/8e3b244973974c318dfcd6663cc274b9 is 50, key is test_row_0/A:col10/1732692284274/Put/seqid=0 2024-11-27T07:24:44,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/92f253966f824daaabf1a2c86598a46f is 50, key is test_row_0/A:col10/1732692284805/Put/seqid=0 2024-11-27T07:24:44,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741918_1094 (size=13221) 2024-11-27T07:24:44,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741919_1095 (size=13221) 2024-11-27T07:24:44,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741920_1096 (size=14741) 2024-11-27T07:24:44,880 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/92f253966f824daaabf1a2c86598a46f 2024-11-27T07:24:44,881 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/8e3b244973974c318dfcd6663cc274b9 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/8e3b244973974c318dfcd6663cc274b9 2024-11-27T07:24:44,893 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 18c429ab939f89ba8ff6429b6bcb0851/A of 18c429ab939f89ba8ff6429b6bcb0851 into 8e3b244973974c318dfcd6663cc274b9(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:24:44,893 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18c429ab939f89ba8ff6429b6bcb0851: 2024-11-27T07:24:44,893 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., storeName=18c429ab939f89ba8ff6429b6bcb0851/A, priority=13, startTime=1732692284773; duration=0sec 2024-11-27T07:24:44,893 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:24:44,893 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18c429ab939f89ba8ff6429b6bcb0851:A 2024-11-27T07:24:44,894 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:24:44,895 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/26a353b813ac45de925a0444eda65108 is 50, key is test_row_0/B:col10/1732692284805/Put/seqid=0 2024-11-27T07:24:44,897 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:24:44,898 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1541): 18c429ab939f89ba8ff6429b6bcb0851/C is initiating minor compaction (all files) 2024-11-27T07:24:44,898 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18c429ab939f89ba8ff6429b6bcb0851/C in TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:44,898 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/483df80637d045e88c6f1b5afc01ad31, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/15da2e53db0141e99d458cc5573156ab, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/262276ecf66b47928edb55c47ab467e1] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp, totalSize=36.8 K 2024-11-27T07:24:44,899 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 483df80637d045e88c6f1b5afc01ad31, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1732692281829 2024-11-27T07:24:44,899 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15da2e53db0141e99d458cc5573156ab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1732692283002 2024-11-27T07:24:44,901 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 262276ecf66b47928edb55c47ab467e1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1732692284240 2024-11-27T07:24:44,924 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18c429ab939f89ba8ff6429b6bcb0851#C#compaction#85 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:24:44,925 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/d17890b4daa9437581a02684c50a45d8 is 50, key is test_row_0/C:col10/1732692284274/Put/seqid=0 2024-11-27T07:24:44,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741921_1097 (size=12301) 2024-11-27T07:24:44,933 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/26a353b813ac45de925a0444eda65108 2024-11-27T07:24:44,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:44,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 314 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50064 deadline: 1732692344938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:44,940 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:44,940 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:44,940 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:44,958 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/489f3735767b403ea21a5c3e8dd88752 is 50, key is test_row_0/C:col10/1732692284805/Put/seqid=0 2024-11-27T07:24:44,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741923_1099 (size=12301) 2024-11-27T07:24:44,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741922_1098 (size=13221) 2024-11-27T07:24:45,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-27T07:24:45,039 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T07:24:45,039 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T07:24:45,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees 2024-11-27T07:24:45,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-27T07:24:45,042 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T07:24:45,043 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T07:24:45,043 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T07:24:45,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:45,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 315 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50064 deadline: 1732692345049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:45,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:45,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:45,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:45,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-27T07:24:45,195 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-27T07:24:45,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:45,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. as already flushing 2024-11-27T07:24:45,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:45,195 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:45,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:45,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:45,243 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/4f4f68ab2be840139abacf50c4eec012 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/4f4f68ab2be840139abacf50c4eec012 2024-11-27T07:24:45,251 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 18c429ab939f89ba8ff6429b6bcb0851/B of 18c429ab939f89ba8ff6429b6bcb0851 into 4f4f68ab2be840139abacf50c4eec012(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:24:45,251 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18c429ab939f89ba8ff6429b6bcb0851: 2024-11-27T07:24:45,254 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., storeName=18c429ab939f89ba8ff6429b6bcb0851/B, priority=13, startTime=1732692284773; duration=0sec 2024-11-27T07:24:45,255 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:24:45,255 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18c429ab939f89ba8ff6429b6bcb0851:B 2024-11-27T07:24:45,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:45,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 316 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50064 deadline: 1732692345259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:45,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:45,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:45,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:45,348 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-27T07:24:45,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:45,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. as already flushing 2024-11-27T07:24:45,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:45,348 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:45,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:45,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:45,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-27T07:24:45,383 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/489f3735767b403ea21a5c3e8dd88752 2024-11-27T07:24:45,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/92f253966f824daaabf1a2c86598a46f as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/92f253966f824daaabf1a2c86598a46f 2024-11-27T07:24:45,400 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/d17890b4daa9437581a02684c50a45d8 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/d17890b4daa9437581a02684c50a45d8 2024-11-27T07:24:45,404 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/92f253966f824daaabf1a2c86598a46f, entries=200, sequenceid=398, filesize=14.4 K 2024-11-27T07:24:45,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/26a353b813ac45de925a0444eda65108 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/26a353b813ac45de925a0444eda65108 2024-11-27T07:24:45,408 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 18c429ab939f89ba8ff6429b6bcb0851/C of 18c429ab939f89ba8ff6429b6bcb0851 into d17890b4daa9437581a02684c50a45d8(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:24:45,408 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18c429ab939f89ba8ff6429b6bcb0851: 2024-11-27T07:24:45,408 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., storeName=18c429ab939f89ba8ff6429b6bcb0851/C, priority=13, startTime=1732692284773; duration=0sec 2024-11-27T07:24:45,408 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:24:45,408 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18c429ab939f89ba8ff6429b6bcb0851:C 2024-11-27T07:24:45,413 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/26a353b813ac45de925a0444eda65108, entries=150, sequenceid=398, filesize=12.0 K 2024-11-27T07:24:45,415 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/489f3735767b403ea21a5c3e8dd88752 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/489f3735767b403ea21a5c3e8dd88752 2024-11-27T07:24:45,424 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/489f3735767b403ea21a5c3e8dd88752, entries=150, sequenceid=398, filesize=12.0 K 2024-11-27T07:24:45,427 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 18c429ab939f89ba8ff6429b6bcb0851 in 619ms, sequenceid=398, compaction requested=false 2024-11-27T07:24:45,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 18c429ab939f89ba8ff6429b6bcb0851: 2024-11-27T07:24:45,507 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-27T07:24:45,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:45,507 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2902): Flushing 18c429ab939f89ba8ff6429b6bcb0851 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-27T07:24:45,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/c6a910d2d14f451f8b7f1a7277838636 is 50, key is test_row_0/A:col10/1732692284926/Put/seqid=0 2024-11-27T07:24:45,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741924_1100 (size=12301) 2024-11-27T07:24:45,538 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/c6a910d2d14f451f8b7f1a7277838636 2024-11-27T07:24:45,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/8528cfe3b97e443d83cce2b4979d2002 is 50, key is test_row_0/B:col10/1732692284926/Put/seqid=0 2024-11-27T07:24:45,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741925_1101 (size=12301) 2024-11-27T07:24:45,558 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/8528cfe3b97e443d83cce2b4979d2002 2024-11-27T07:24:45,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 18c429ab939f89ba8ff6429b6bcb0851 2024-11-27T07:24:45,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. as already flushing 2024-11-27T07:24:45,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/4e9dcccee88c4d51825b7233029da5dc is 50, key is test_row_0/C:col10/1732692284926/Put/seqid=0 2024-11-27T07:24:45,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741926_1102 (size=12301) 2024-11-27T07:24:45,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:45,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 326 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50064 deadline: 1732692345602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:45,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:45,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:45,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:45,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-27T07:24:45,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:45,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 327 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50064 deadline: 1732692345709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:45,712 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:45,712 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:45,712 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:45,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:45,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 328 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50064 deadline: 1732692345919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:45,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:45,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:45,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:24:46,001 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=425 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/4e9dcccee88c4d51825b7233029da5dc 2024-11-27T07:24:46,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/c6a910d2d14f451f8b7f1a7277838636 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/c6a910d2d14f451f8b7f1a7277838636 2024-11-27T07:24:46,010 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/c6a910d2d14f451f8b7f1a7277838636, entries=150, sequenceid=425, filesize=12.0 K 2024-11-27T07:24:46,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/8528cfe3b97e443d83cce2b4979d2002 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/8528cfe3b97e443d83cce2b4979d2002 2024-11-27T07:24:46,016 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/8528cfe3b97e443d83cce2b4979d2002, entries=150, sequenceid=425, filesize=12.0 K 2024-11-27T07:24:46,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/4e9dcccee88c4d51825b7233029da5dc as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/4e9dcccee88c4d51825b7233029da5dc 2024-11-27T07:24:46,023 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/4e9dcccee88c4d51825b7233029da5dc, entries=150, sequenceid=425, filesize=12.0 K 2024-11-27T07:24:46,025 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 18c429ab939f89ba8ff6429b6bcb0851 in 517ms, sequenceid=425, compaction requested=true 2024-11-27T07:24:46,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2603): Flush status journal for 18c429ab939f89ba8ff6429b6bcb0851: 2024-11-27T07:24:46,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:46,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=28 2024-11-27T07:24:46,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster(4169): Remote procedure done, pid=28 2024-11-27T07:24:46,028 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-27T07:24:46,029 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 983 msec 2024-11-27T07:24:46,031 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees in 990 msec 2024-11-27T07:24:46,099 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-27T07:24:46,099 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-27T07:24:46,099 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T07:24:46,099 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T07:24:46,099 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-27T07:24:46,100 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T07:24:46,102 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-27T07:24:46,102 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T07:24:46,104 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:24:46,104 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:24:46,104 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:24:46,104 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:24:46,105 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:24:46,105 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:24:46,105 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T07:24:46,105 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T07:24:46,105 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:24:46,105 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T07:24:46,105 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:24:46,106 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T07:24:46,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-27T07:24:46,180 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T07:24:46,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 18c429ab939f89ba8ff6429b6bcb0851 2024-11-27T07:24:46,231 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T07:24:46,231 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 18c429ab939f89ba8ff6429b6bcb0851 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-27T07:24:46,231 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-27T07:24:46,231 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T07:24:46,231 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:24:46,231 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:24:46,231 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T07:24:46,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/9b9de871f920413e94350449d5463d53 is 50, key is test_row_0/A:col10/1732692285593/Put/seqid=0 2024-11-27T07:24:46,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741927_1103 (size=12301) 2024-11-27T07:24:46,644 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/9b9de871f920413e94350449d5463d53 2024-11-27T07:24:46,656 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/280700893e9c48c9abb6768a5003beba is 50, key is test_row_0/B:col10/1732692285593/Put/seqid=0 2024-11-27T07:24:46,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741928_1104 (size=12301) 2024-11-27T07:24:47,062 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/280700893e9c48c9abb6768a5003beba 2024-11-27T07:24:47,075 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/77d35562ed5149c088bd777bb9b27be5 is 50, key is test_row_0/C:col10/1732692285593/Put/seqid=0 2024-11-27T07:24:47,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741929_1105 (size=12301) 2024-11-27T07:24:47,480 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/77d35562ed5149c088bd777bb9b27be5 2024-11-27T07:24:47,541 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/9b9de871f920413e94350449d5463d53 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/9b9de871f920413e94350449d5463d53 2024-11-27T07:24:47,549 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/9b9de871f920413e94350449d5463d53, entries=150, sequenceid=438, filesize=12.0 K 2024-11-27T07:24:47,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/280700893e9c48c9abb6768a5003beba as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/280700893e9c48c9abb6768a5003beba 2024-11-27T07:24:47,555 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/280700893e9c48c9abb6768a5003beba, entries=150, sequenceid=438, filesize=12.0 K 2024-11-27T07:24:47,556 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/77d35562ed5149c088bd777bb9b27be5 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/77d35562ed5149c088bd777bb9b27be5 2024-11-27T07:24:47,561 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/77d35562ed5149c088bd777bb9b27be5, entries=150, sequenceid=438, filesize=12.0 K 2024-11-27T07:24:47,562 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=0 B/0 for 18c429ab939f89ba8ff6429b6bcb0851 in 1331ms, sequenceid=438, compaction requested=true 2024-11-27T07:24:47,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 18c429ab939f89ba8ff6429b6bcb0851: 2024-11-27T07:24:47,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18c429ab939f89ba8ff6429b6bcb0851:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T07:24:47,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:24:47,562 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T07:24:47,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18c429ab939f89ba8ff6429b6bcb0851:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T07:24:47,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:24:47,562 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T07:24:47,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18c429ab939f89ba8ff6429b6bcb0851:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T07:24:47,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:24:47,563 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T07:24:47,563 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52564 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T07:24:47,563 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1541): 18c429ab939f89ba8ff6429b6bcb0851/A is initiating minor compaction (all files) 2024-11-27T07:24:47,563 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 18c429ab939f89ba8ff6429b6bcb0851/B is initiating minor compaction (all files) 2024-11-27T07:24:47,564 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18c429ab939f89ba8ff6429b6bcb0851/B in TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:47,564 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18c429ab939f89ba8ff6429b6bcb0851/A in TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:47,564 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/4f4f68ab2be840139abacf50c4eec012, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/26a353b813ac45de925a0444eda65108, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/8528cfe3b97e443d83cce2b4979d2002, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/280700893e9c48c9abb6768a5003beba] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp, totalSize=48.9 K 2024-11-27T07:24:47,564 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/8e3b244973974c318dfcd6663cc274b9, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/92f253966f824daaabf1a2c86598a46f, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/c6a910d2d14f451f8b7f1a7277838636, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/9b9de871f920413e94350449d5463d53] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp, totalSize=51.3 K 2024-11-27T07:24:47,564 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e3b244973974c318dfcd6663cc274b9, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1732692284240 2024-11-27T07:24:47,564 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f4f68ab2be840139abacf50c4eec012, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1732692284240 2024-11-27T07:24:47,564 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92f253966f824daaabf1a2c86598a46f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1732692284630 2024-11-27T07:24:47,564 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 26a353b813ac45de925a0444eda65108, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1732692284630 2024-11-27T07:24:47,565 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6a910d2d14f451f8b7f1a7277838636, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1732692284920 2024-11-27T07:24:47,565 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 8528cfe3b97e443d83cce2b4979d2002, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1732692284920 2024-11-27T07:24:47,565 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b9de871f920413e94350449d5463d53, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1732692285593 2024-11-27T07:24:47,565 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 280700893e9c48c9abb6768a5003beba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1732692285593 2024-11-27T07:24:47,573 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18c429ab939f89ba8ff6429b6bcb0851#B#compaction#93 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:24:47,573 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18c429ab939f89ba8ff6429b6bcb0851#A#compaction#94 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:24:47,574 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/2f585c6cd189480d8fccf1e653999413 is 50, key is test_row_0/B:col10/1732692285593/Put/seqid=0 2024-11-27T07:24:47,574 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/7eecfe06648841ff817b83bf3a92a3f3 is 50, key is test_row_0/A:col10/1732692285593/Put/seqid=0 2024-11-27T07:24:47,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741931_1107 (size=13357) 2024-11-27T07:24:47,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741930_1106 (size=13357) 2024-11-27T07:24:47,969 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-27T07:24:47,989 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/2f585c6cd189480d8fccf1e653999413 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/2f585c6cd189480d8fccf1e653999413 2024-11-27T07:24:47,989 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/7eecfe06648841ff817b83bf3a92a3f3 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/7eecfe06648841ff817b83bf3a92a3f3 2024-11-27T07:24:47,994 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 18c429ab939f89ba8ff6429b6bcb0851/B of 18c429ab939f89ba8ff6429b6bcb0851 into 2f585c6cd189480d8fccf1e653999413(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:24:47,994 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 18c429ab939f89ba8ff6429b6bcb0851/A of 18c429ab939f89ba8ff6429b6bcb0851 into 7eecfe06648841ff817b83bf3a92a3f3(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:24:47,994 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18c429ab939f89ba8ff6429b6bcb0851: 2024-11-27T07:24:47,994 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18c429ab939f89ba8ff6429b6bcb0851: 2024-11-27T07:24:47,994 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., storeName=18c429ab939f89ba8ff6429b6bcb0851/B, priority=12, startTime=1732692287562; duration=0sec 2024-11-27T07:24:47,994 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., storeName=18c429ab939f89ba8ff6429b6bcb0851/A, priority=12, startTime=1732692287562; duration=0sec 2024-11-27T07:24:47,995 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:24:47,995 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18c429ab939f89ba8ff6429b6bcb0851:B 2024-11-27T07:24:47,995 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:24:47,995 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T07:24:47,995 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18c429ab939f89ba8ff6429b6bcb0851:A 2024-11-27T07:24:47,996 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T07:24:47,996 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 18c429ab939f89ba8ff6429b6bcb0851/C is initiating minor compaction (all files) 2024-11-27T07:24:47,996 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18c429ab939f89ba8ff6429b6bcb0851/C in TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:47,996 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/d17890b4daa9437581a02684c50a45d8, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/489f3735767b403ea21a5c3e8dd88752, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/4e9dcccee88c4d51825b7233029da5dc, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/77d35562ed5149c088bd777bb9b27be5] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp, totalSize=48.9 K 2024-11-27T07:24:47,997 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting d17890b4daa9437581a02684c50a45d8, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1732692284240 2024-11-27T07:24:47,997 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 489f3735767b403ea21a5c3e8dd88752, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1732692284630 2024-11-27T07:24:47,997 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e9dcccee88c4d51825b7233029da5dc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=425, earliestPutTs=1732692284920 2024-11-27T07:24:47,998 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 77d35562ed5149c088bd777bb9b27be5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1732692285593 2024-11-27T07:24:48,006 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18c429ab939f89ba8ff6429b6bcb0851#C#compaction#95 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:24:48,006 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/ffa0900560fe4b3b9de9ab6da1292d5c is 50, key is test_row_0/C:col10/1732692285593/Put/seqid=0 2024-11-27T07:24:48,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741932_1108 (size=13357) 2024-11-27T07:24:48,420 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/ffa0900560fe4b3b9de9ab6da1292d5c as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/ffa0900560fe4b3b9de9ab6da1292d5c 2024-11-27T07:24:48,427 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 18c429ab939f89ba8ff6429b6bcb0851/C of 18c429ab939f89ba8ff6429b6bcb0851 into ffa0900560fe4b3b9de9ab6da1292d5c(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:24:48,427 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18c429ab939f89ba8ff6429b6bcb0851: 2024-11-27T07:24:48,427 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., storeName=18c429ab939f89ba8ff6429b6bcb0851/C, priority=12, startTime=1732692287562; duration=0sec 2024-11-27T07:24:48,427 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:24:48,427 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18c429ab939f89ba8ff6429b6bcb0851:C 2024-11-27T07:24:49,371 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-27T07:24:49,371 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T07:24:49,371 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:24:49,371 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:24:49,372 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T07:24:49,382 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-27T07:24:49,383 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T07:24:49,383 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:24:49,383 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:24:49,383 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T07:24:49,432 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-27T07:24:49,432 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T07:24:49,433 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:24:49,433 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:24:49,433 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T07:24:49,472 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-27T07:24:49,472 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T07:24:49,472 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:24:49,472 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:24:49,472 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T07:24:49,473 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-27T07:24:49,473 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 20 2024-11-27T07:24:49,473 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 24 2024-11-27T07:24:49,473 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 18 2024-11-27T07:24:49,473 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 20 2024-11-27T07:24:49,473 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 266 2024-11-27T07:24:49,473 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-27T07:24:49,473 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3449 2024-11-27T07:24:49,473 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3362 2024-11-27T07:24:49,473 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-27T07:24:49,474 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1567 2024-11-27T07:24:49,474 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4701 rows 2024-11-27T07:24:49,474 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1547 2024-11-27T07:24:49,474 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4641 rows 2024-11-27T07:24:49,474 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T07:24:49,474 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T07:24:49,474 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:24:49,475 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:24:49,475 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T07:24:49,482 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-27T07:24:49,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-27T07:24:49,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-27T07:24:49,491 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732692289491"}]},"ts":"1732692289491"} 2024-11-27T07:24:49,493 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-27T07:24:49,493 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-27T07:24:49,495 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-27T07:24:49,498 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=18c429ab939f89ba8ff6429b6bcb0851, UNASSIGN}] 2024-11-27T07:24:49,499 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=18c429ab939f89ba8ff6429b6bcb0851, UNASSIGN 2024-11-27T07:24:49,500 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=18c429ab939f89ba8ff6429b6bcb0851, regionState=CLOSING, regionLocation=289954c9202d,42117,1732692259671 2024-11-27T07:24:49,502 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43057 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=289954c9202d,42117,1732692259671, table=TestAcidGuarantees, region=18c429ab939f89ba8ff6429b6bcb0851. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-27T07:24:49,503 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=18c429ab939f89ba8ff6429b6bcb0851, UNASSIGN because future has completed 2024-11-27T07:24:49,503 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T07:24:49,504 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure 18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671}] 2024-11-27T07:24:49,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-27T07:24:49,661 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(122): Close 18c429ab939f89ba8ff6429b6bcb0851 2024-11-27T07:24:49,661 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T07:24:49,662 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1722): Closing 18c429ab939f89ba8ff6429b6bcb0851, disabling compactions & flushes 2024-11-27T07:24:49,662 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:49,662 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:49,662 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. after waiting 0 ms 2024-11-27T07:24:49,662 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:49,662 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(2902): Flushing 18c429ab939f89ba8ff6429b6bcb0851 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-27T07:24:49,666 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/7bec5ea4952d45518f6543c631541882 is 50, key is test_row_0/A:col10/1732692289380/Put/seqid=0 2024-11-27T07:24:49,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741933_1109 (size=12301) 2024-11-27T07:24:49,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-27T07:24:50,072 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=448 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/7bec5ea4952d45518f6543c631541882 2024-11-27T07:24:50,087 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/f113f39bfd294ff281b05ba799fa15b6 is 50, key is test_row_0/B:col10/1732692289380/Put/seqid=0 2024-11-27T07:24:50,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741934_1110 (size=12301) 2024-11-27T07:24:50,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-27T07:24:50,493 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=448 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/f113f39bfd294ff281b05ba799fa15b6 2024-11-27T07:24:50,508 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/8a8a9eb51d384eaf8346de2a7d7bed4e is 50, key is test_row_0/C:col10/1732692289380/Put/seqid=0 2024-11-27T07:24:50,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741935_1111 (size=12301) 2024-11-27T07:24:50,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-27T07:24:50,915 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=448 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/8a8a9eb51d384eaf8346de2a7d7bed4e 2024-11-27T07:24:50,928 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/A/7bec5ea4952d45518f6543c631541882 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/7bec5ea4952d45518f6543c631541882 2024-11-27T07:24:50,934 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/7bec5ea4952d45518f6543c631541882, entries=150, sequenceid=448, filesize=12.0 K 2024-11-27T07:24:50,936 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/B/f113f39bfd294ff281b05ba799fa15b6 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/f113f39bfd294ff281b05ba799fa15b6 2024-11-27T07:24:50,941 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/f113f39bfd294ff281b05ba799fa15b6, entries=150, sequenceid=448, filesize=12.0 K 2024-11-27T07:24:50,943 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/.tmp/C/8a8a9eb51d384eaf8346de2a7d7bed4e as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/8a8a9eb51d384eaf8346de2a7d7bed4e 2024-11-27T07:24:50,948 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/8a8a9eb51d384eaf8346de2a7d7bed4e, entries=150, sequenceid=448, filesize=12.0 K 2024-11-27T07:24:50,949 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 18c429ab939f89ba8ff6429b6bcb0851 in 1287ms, sequenceid=448, compaction requested=false 2024-11-27T07:24:50,950 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/bb61adad26b6438ba077c01ef4ae3ab0, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/2771d35241304596af2a22bf03120038, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/e2ab7e3672db45c1b6bef1d9e2ae420c, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/df79fcd76ea3485980a36cd1cd00c119, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/235d772a54ef495d856e72d2f4a2314d, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/8ec64fce945d4655b8c2d178adc1ecc7, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/0771928f1e1842b6a9fd37fbca6d184c, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/d04fac4f2bd6432e9fe7d2ce8bba4717, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/9a8b745bc9894c61b15cc8d334a06060, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/d51a7bb5825c4c908d13430a9fb61b09, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/ecc6c9ee8cda46ed957710dd7c0d1068, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/87c8c972c53747a090b431f2392d7847, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/30df5ea5072946f4834f08170594efa6, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/9c55ea41112440d9b90dca7ccd6c33fd, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/a7a35f378fb740f386214dea42606228, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/a41b91706baa41e88d69da86feed98b8, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/42bf3dfcfe0d4524b965446349e3a602, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/f3d01c3e5df94102aa361838d6a85684, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/3ddbabbdc54b47548d1fa926743645e8, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/a5351e8fbc9947cdae9a31b434b3b989, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/61a5df9c28c043ba9aa8443d06363d12, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/9f4861af7ce048f3ac81ccbfebb0bf1a, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/d6cdd47e7f344a3b81ed9986042487e8, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/6d7d250887d54274929c21d5eaae0ba9, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/fb6240e37c014f099e895c3915c52ae5, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/d656e0c13e494a699c4a3b969318f612, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/8e3b244973974c318dfcd6663cc274b9, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/d210d4bdebf9444ab914cda2d38b3adc, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/92f253966f824daaabf1a2c86598a46f, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/c6a910d2d14f451f8b7f1a7277838636, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/9b9de871f920413e94350449d5463d53] to archive 2024-11-27T07:24:50,953 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T07:24:50,958 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/bb61adad26b6438ba077c01ef4ae3ab0 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/bb61adad26b6438ba077c01ef4ae3ab0 2024-11-27T07:24:50,959 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/2771d35241304596af2a22bf03120038 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/2771d35241304596af2a22bf03120038 2024-11-27T07:24:50,961 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/e2ab7e3672db45c1b6bef1d9e2ae420c to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/e2ab7e3672db45c1b6bef1d9e2ae420c 2024-11-27T07:24:50,962 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/df79fcd76ea3485980a36cd1cd00c119 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/df79fcd76ea3485980a36cd1cd00c119 2024-11-27T07:24:50,963 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/235d772a54ef495d856e72d2f4a2314d to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/235d772a54ef495d856e72d2f4a2314d 2024-11-27T07:24:50,965 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/8ec64fce945d4655b8c2d178adc1ecc7 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/8ec64fce945d4655b8c2d178adc1ecc7 2024-11-27T07:24:50,966 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/0771928f1e1842b6a9fd37fbca6d184c to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/0771928f1e1842b6a9fd37fbca6d184c 2024-11-27T07:24:50,967 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/d04fac4f2bd6432e9fe7d2ce8bba4717 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/d04fac4f2bd6432e9fe7d2ce8bba4717 2024-11-27T07:24:50,969 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/9a8b745bc9894c61b15cc8d334a06060 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/9a8b745bc9894c61b15cc8d334a06060 2024-11-27T07:24:50,970 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/d51a7bb5825c4c908d13430a9fb61b09 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/d51a7bb5825c4c908d13430a9fb61b09 2024-11-27T07:24:50,971 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/ecc6c9ee8cda46ed957710dd7c0d1068 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/ecc6c9ee8cda46ed957710dd7c0d1068 2024-11-27T07:24:50,973 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/87c8c972c53747a090b431f2392d7847 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/87c8c972c53747a090b431f2392d7847 2024-11-27T07:24:50,974 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/30df5ea5072946f4834f08170594efa6 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/30df5ea5072946f4834f08170594efa6 2024-11-27T07:24:50,975 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/9c55ea41112440d9b90dca7ccd6c33fd to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/9c55ea41112440d9b90dca7ccd6c33fd 2024-11-27T07:24:50,977 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/a7a35f378fb740f386214dea42606228 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/a7a35f378fb740f386214dea42606228 2024-11-27T07:24:50,978 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/a41b91706baa41e88d69da86feed98b8 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/a41b91706baa41e88d69da86feed98b8 2024-11-27T07:24:50,980 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/42bf3dfcfe0d4524b965446349e3a602 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/42bf3dfcfe0d4524b965446349e3a602 2024-11-27T07:24:50,982 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/f3d01c3e5df94102aa361838d6a85684 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/f3d01c3e5df94102aa361838d6a85684 2024-11-27T07:24:50,983 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/3ddbabbdc54b47548d1fa926743645e8 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/3ddbabbdc54b47548d1fa926743645e8 2024-11-27T07:24:50,985 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/a5351e8fbc9947cdae9a31b434b3b989 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/a5351e8fbc9947cdae9a31b434b3b989 2024-11-27T07:24:50,986 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/61a5df9c28c043ba9aa8443d06363d12 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/61a5df9c28c043ba9aa8443d06363d12 2024-11-27T07:24:50,987 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/9f4861af7ce048f3ac81ccbfebb0bf1a to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/9f4861af7ce048f3ac81ccbfebb0bf1a 2024-11-27T07:24:50,989 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/d6cdd47e7f344a3b81ed9986042487e8 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/d6cdd47e7f344a3b81ed9986042487e8 2024-11-27T07:24:50,990 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/6d7d250887d54274929c21d5eaae0ba9 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/6d7d250887d54274929c21d5eaae0ba9 2024-11-27T07:24:50,991 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/fb6240e37c014f099e895c3915c52ae5 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/fb6240e37c014f099e895c3915c52ae5 2024-11-27T07:24:50,992 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/d656e0c13e494a699c4a3b969318f612 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/d656e0c13e494a699c4a3b969318f612 2024-11-27T07:24:50,996 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/8e3b244973974c318dfcd6663cc274b9 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/8e3b244973974c318dfcd6663cc274b9 2024-11-27T07:24:50,998 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/d210d4bdebf9444ab914cda2d38b3adc to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/d210d4bdebf9444ab914cda2d38b3adc 2024-11-27T07:24:51,000 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/92f253966f824daaabf1a2c86598a46f to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/92f253966f824daaabf1a2c86598a46f 2024-11-27T07:24:51,001 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/c6a910d2d14f451f8b7f1a7277838636 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/c6a910d2d14f451f8b7f1a7277838636 2024-11-27T07:24:51,002 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/9b9de871f920413e94350449d5463d53 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/9b9de871f920413e94350449d5463d53 2024-11-27T07:24:51,014 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/37ad452ef6c240f49b3d5a2538072886, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/11707fad122c46a6a3348039f9f20a82, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/870d9ccc93a74fccb6a29016d70b2663, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/e3d25f8da2374a64b547a7528eecf9eb, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/a27360316ddf4cb6a03a2386a0327527, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/37723e1c6944487f9d8953338db3fbc5, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/b49eb9d3ba73439aa95c84f989b57e92, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/49482dde3e574e8bb9cee3bc38557db0, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/42b329dea6a543c28685aca7724a457d, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/a1e46af630fd4680af93df8d585f7b07, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/5488d1fe32c4420f8f62607c9b800b62, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/8c049c9492a34f37bbee7dc9a00b9f78, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/3d4da3171a4a4f3eb05197f69fd56b2c, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/b2e0de61f0ae430285ba091c4d65ad56, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/7eeb849dd6394c42801a55a6a6d8c9c1, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/73a04026511c4c4382082776dffe6c68, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/983434dc03bb4f7b99277779cc60705d, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/11f78494130146bb86af212c6b85c0ea, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/684329091a684328b3ced5c80038cd31, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/5aaa1143a1ad4050901f029d90685999, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/4f80d7236c5d466a8b77b190717860e7, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/baae975a28094037ba4cd653541b4797, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/a0c66d9c89a243d4921a063143d0c5ea, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/8016129177b046a299835e2a81f16168, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/70d05ab955cf4ea19384397a656400e5, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/c28d852f7f034c34a185e80d5caca9c5, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/4f4f68ab2be840139abacf50c4eec012, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/0f565176e5804256b5a70623de2faa12, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/26a353b813ac45de925a0444eda65108, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/8528cfe3b97e443d83cce2b4979d2002, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/280700893e9c48c9abb6768a5003beba] to archive 2024-11-27T07:24:51,015 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T07:24:51,017 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/37ad452ef6c240f49b3d5a2538072886 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/37ad452ef6c240f49b3d5a2538072886 2024-11-27T07:24:51,018 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/11707fad122c46a6a3348039f9f20a82 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/11707fad122c46a6a3348039f9f20a82 2024-11-27T07:24:51,019 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/870d9ccc93a74fccb6a29016d70b2663 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/870d9ccc93a74fccb6a29016d70b2663 2024-11-27T07:24:51,020 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/e3d25f8da2374a64b547a7528eecf9eb to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/e3d25f8da2374a64b547a7528eecf9eb 2024-11-27T07:24:51,022 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/a27360316ddf4cb6a03a2386a0327527 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/a27360316ddf4cb6a03a2386a0327527 2024-11-27T07:24:51,023 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/37723e1c6944487f9d8953338db3fbc5 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/37723e1c6944487f9d8953338db3fbc5 2024-11-27T07:24:51,024 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/b49eb9d3ba73439aa95c84f989b57e92 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/b49eb9d3ba73439aa95c84f989b57e92 2024-11-27T07:24:51,026 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/49482dde3e574e8bb9cee3bc38557db0 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/49482dde3e574e8bb9cee3bc38557db0 2024-11-27T07:24:51,027 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/42b329dea6a543c28685aca7724a457d to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/42b329dea6a543c28685aca7724a457d 2024-11-27T07:24:51,029 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/a1e46af630fd4680af93df8d585f7b07 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/a1e46af630fd4680af93df8d585f7b07 2024-11-27T07:24:51,030 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/5488d1fe32c4420f8f62607c9b800b62 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/5488d1fe32c4420f8f62607c9b800b62 2024-11-27T07:24:51,031 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/8c049c9492a34f37bbee7dc9a00b9f78 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/8c049c9492a34f37bbee7dc9a00b9f78 2024-11-27T07:24:51,032 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/3d4da3171a4a4f3eb05197f69fd56b2c to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/3d4da3171a4a4f3eb05197f69fd56b2c 2024-11-27T07:24:51,033 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/b2e0de61f0ae430285ba091c4d65ad56 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/b2e0de61f0ae430285ba091c4d65ad56 2024-11-27T07:24:51,034 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/7eeb849dd6394c42801a55a6a6d8c9c1 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/7eeb849dd6394c42801a55a6a6d8c9c1 2024-11-27T07:24:51,034 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/73a04026511c4c4382082776dffe6c68 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/73a04026511c4c4382082776dffe6c68 2024-11-27T07:24:51,035 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/983434dc03bb4f7b99277779cc60705d to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/983434dc03bb4f7b99277779cc60705d 2024-11-27T07:24:51,036 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/11f78494130146bb86af212c6b85c0ea to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/11f78494130146bb86af212c6b85c0ea 2024-11-27T07:24:51,037 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/684329091a684328b3ced5c80038cd31 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/684329091a684328b3ced5c80038cd31 2024-11-27T07:24:51,038 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/5aaa1143a1ad4050901f029d90685999 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/5aaa1143a1ad4050901f029d90685999 2024-11-27T07:24:51,039 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/4f80d7236c5d466a8b77b190717860e7 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/4f80d7236c5d466a8b77b190717860e7 2024-11-27T07:24:51,040 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/baae975a28094037ba4cd653541b4797 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/baae975a28094037ba4cd653541b4797 2024-11-27T07:24:51,041 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/a0c66d9c89a243d4921a063143d0c5ea to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/a0c66d9c89a243d4921a063143d0c5ea 2024-11-27T07:24:51,042 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/8016129177b046a299835e2a81f16168 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/8016129177b046a299835e2a81f16168 2024-11-27T07:24:51,043 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/70d05ab955cf4ea19384397a656400e5 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/70d05ab955cf4ea19384397a656400e5 2024-11-27T07:24:51,044 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/c28d852f7f034c34a185e80d5caca9c5 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/c28d852f7f034c34a185e80d5caca9c5 2024-11-27T07:24:51,045 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/4f4f68ab2be840139abacf50c4eec012 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/4f4f68ab2be840139abacf50c4eec012 2024-11-27T07:24:51,047 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/0f565176e5804256b5a70623de2faa12 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/0f565176e5804256b5a70623de2faa12 2024-11-27T07:24:51,048 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/26a353b813ac45de925a0444eda65108 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/26a353b813ac45de925a0444eda65108 2024-11-27T07:24:51,049 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/8528cfe3b97e443d83cce2b4979d2002 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/8528cfe3b97e443d83cce2b4979d2002 2024-11-27T07:24:51,050 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/280700893e9c48c9abb6768a5003beba to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/280700893e9c48c9abb6768a5003beba 2024-11-27T07:24:51,051 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/7f93e74822e24e3682298e271de742f0, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/8313105513aa44379a12b308b31cb3ff, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/a972dcd16d4e46128b3cbc3c8dfbeed4, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/d9b06e2cf8814d659d91071e5517906e, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/0c8d7c2dddcb4b14ac6b9225b6960abc, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/d76d5ff811084c188c00a95f0a1d4033, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/a675df890e544088b436b089046d33d7, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/840cf06b1fbf481d89b11637ec71caef, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/e96dc4b20bd84f098d5ff8d3599f6859, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/a00924dc2f034e94831bf9535d415198, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/bcf5949156f343f1ae4359ffe9cada4a, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/d781860a378844fcaf536a58c8da5b72, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/f39371934e4c446cbd3ce70fa5691aad, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/4712da00af8942b5a80f288361bbcab4, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/145a808f2498471ca40626fc8b0da801, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/3428b337fc464c218dcf1e7542aefd71, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/d756611ecf384003b98d02161399b974, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/dd6b31f4fada4d02a5a7b2960bf45cf8, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/53fbdd21b3ef4ca48bda4b6ac0a8d042, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/0c3e5d08e1a74ea6bfdedbfbafeb665a, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/381b4e0fca1544a5b9a36b7d17b77b73, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/9e9d7bd4556a4d6ba9c4b31fc1296a4d, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/8a3728f7820f4fc5ac64f985496de9c5, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/483df80637d045e88c6f1b5afc01ad31, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/314306bc145a4f4da646de9e73abd915, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/15da2e53db0141e99d458cc5573156ab, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/d17890b4daa9437581a02684c50a45d8, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/262276ecf66b47928edb55c47ab467e1, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/489f3735767b403ea21a5c3e8dd88752, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/4e9dcccee88c4d51825b7233029da5dc, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/77d35562ed5149c088bd777bb9b27be5] to archive 2024-11-27T07:24:51,052 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T07:24:51,054 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/7f93e74822e24e3682298e271de742f0 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/7f93e74822e24e3682298e271de742f0 2024-11-27T07:24:51,055 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/8313105513aa44379a12b308b31cb3ff to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/8313105513aa44379a12b308b31cb3ff 2024-11-27T07:24:51,056 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/a972dcd16d4e46128b3cbc3c8dfbeed4 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/a972dcd16d4e46128b3cbc3c8dfbeed4 2024-11-27T07:24:51,057 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/d9b06e2cf8814d659d91071e5517906e to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/d9b06e2cf8814d659d91071e5517906e 2024-11-27T07:24:51,059 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/0c8d7c2dddcb4b14ac6b9225b6960abc to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/0c8d7c2dddcb4b14ac6b9225b6960abc 2024-11-27T07:24:51,060 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/d76d5ff811084c188c00a95f0a1d4033 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/d76d5ff811084c188c00a95f0a1d4033 2024-11-27T07:24:51,061 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/a675df890e544088b436b089046d33d7 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/a675df890e544088b436b089046d33d7 2024-11-27T07:24:51,062 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/840cf06b1fbf481d89b11637ec71caef to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/840cf06b1fbf481d89b11637ec71caef 2024-11-27T07:24:51,063 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/e96dc4b20bd84f098d5ff8d3599f6859 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/e96dc4b20bd84f098d5ff8d3599f6859 2024-11-27T07:24:51,064 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/a00924dc2f034e94831bf9535d415198 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/a00924dc2f034e94831bf9535d415198 2024-11-27T07:24:51,065 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/bcf5949156f343f1ae4359ffe9cada4a to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/bcf5949156f343f1ae4359ffe9cada4a 2024-11-27T07:24:51,066 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/d781860a378844fcaf536a58c8da5b72 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/d781860a378844fcaf536a58c8da5b72 2024-11-27T07:24:51,067 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/f39371934e4c446cbd3ce70fa5691aad to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/f39371934e4c446cbd3ce70fa5691aad 2024-11-27T07:24:51,068 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/4712da00af8942b5a80f288361bbcab4 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/4712da00af8942b5a80f288361bbcab4 2024-11-27T07:24:51,069 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/145a808f2498471ca40626fc8b0da801 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/145a808f2498471ca40626fc8b0da801 2024-11-27T07:24:51,071 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/3428b337fc464c218dcf1e7542aefd71 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/3428b337fc464c218dcf1e7542aefd71 2024-11-27T07:24:51,072 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/d756611ecf384003b98d02161399b974 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/d756611ecf384003b98d02161399b974 2024-11-27T07:24:51,073 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/dd6b31f4fada4d02a5a7b2960bf45cf8 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/dd6b31f4fada4d02a5a7b2960bf45cf8 2024-11-27T07:24:51,074 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/53fbdd21b3ef4ca48bda4b6ac0a8d042 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/53fbdd21b3ef4ca48bda4b6ac0a8d042 2024-11-27T07:24:51,075 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/0c3e5d08e1a74ea6bfdedbfbafeb665a to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/0c3e5d08e1a74ea6bfdedbfbafeb665a 2024-11-27T07:24:51,076 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/381b4e0fca1544a5b9a36b7d17b77b73 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/381b4e0fca1544a5b9a36b7d17b77b73 2024-11-27T07:24:51,077 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/9e9d7bd4556a4d6ba9c4b31fc1296a4d to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/9e9d7bd4556a4d6ba9c4b31fc1296a4d 2024-11-27T07:24:51,078 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/8a3728f7820f4fc5ac64f985496de9c5 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/8a3728f7820f4fc5ac64f985496de9c5 2024-11-27T07:24:51,079 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/483df80637d045e88c6f1b5afc01ad31 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/483df80637d045e88c6f1b5afc01ad31 2024-11-27T07:24:51,081 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/314306bc145a4f4da646de9e73abd915 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/314306bc145a4f4da646de9e73abd915 2024-11-27T07:24:51,082 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/15da2e53db0141e99d458cc5573156ab to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/15da2e53db0141e99d458cc5573156ab 2024-11-27T07:24:51,083 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/d17890b4daa9437581a02684c50a45d8 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/d17890b4daa9437581a02684c50a45d8 2024-11-27T07:24:51,084 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/262276ecf66b47928edb55c47ab467e1 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/262276ecf66b47928edb55c47ab467e1 2024-11-27T07:24:51,085 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/489f3735767b403ea21a5c3e8dd88752 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/489f3735767b403ea21a5c3e8dd88752 2024-11-27T07:24:51,087 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/4e9dcccee88c4d51825b7233029da5dc to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/4e9dcccee88c4d51825b7233029da5dc 2024-11-27T07:24:51,088 DEBUG [StoreCloser-TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/77d35562ed5149c088bd777bb9b27be5 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/77d35562ed5149c088bd777bb9b27be5 2024-11-27T07:24:51,093 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/recovered.edits/451.seqid, newMaxSeqId=451, maxSeqId=1 2024-11-27T07:24:51,094 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. 2024-11-27T07:24:51,094 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1676): Region close journal for 18c429ab939f89ba8ff6429b6bcb0851: Waiting for close lock at 1732692289662Running coprocessor pre-close hooks at 1732692289662Disabling compacts and flushes for region at 1732692289662Disabling writes for close at 1732692289662Obtaining lock to block concurrent updates at 1732692289662Preparing flush snapshotting stores in 18c429ab939f89ba8ff6429b6bcb0851 at 1732692289662Finished memstore snapshotting TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1732692289663 (+1 ms)Flushing stores of TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851. at 1732692289663Flushing 18c429ab939f89ba8ff6429b6bcb0851/A: creating writer at 1732692289663Flushing 18c429ab939f89ba8ff6429b6bcb0851/A: appending metadata at 1732692289666 (+3 ms)Flushing 18c429ab939f89ba8ff6429b6bcb0851/A: closing flushed file at 1732692289666Flushing 18c429ab939f89ba8ff6429b6bcb0851/B: creating writer at 1732692290084 (+418 ms)Flushing 18c429ab939f89ba8ff6429b6bcb0851/B: appending metadata at 1732692290087 (+3 ms)Flushing 18c429ab939f89ba8ff6429b6bcb0851/B: closing flushed file at 1732692290087Flushing 18c429ab939f89ba8ff6429b6bcb0851/C: creating writer at 1732692290504 (+417 ms)Flushing 18c429ab939f89ba8ff6429b6bcb0851/C: appending metadata at 1732692290508 (+4 ms)Flushing 18c429ab939f89ba8ff6429b6bcb0851/C: closing flushed file at 1732692290508Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@dc301e5: reopening flushed file at 1732692290926 (+418 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@577dd37e: reopening flushed file at 1732692290935 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@33e93ac0: reopening flushed file at 1732692290942 (+7 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 18c429ab939f89ba8ff6429b6bcb0851 in 1287ms, sequenceid=448, compaction requested=false at 1732692290949 (+7 ms)Writing region close event to WAL at 1732692291089 (+140 ms)Running coprocessor post-close hooks at 1732692291093 (+4 ms)Closed at 1732692291094 (+1 ms) 2024-11-27T07:24:51,096 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(157): Closed 18c429ab939f89ba8ff6429b6bcb0851 2024-11-27T07:24:51,096 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=18c429ab939f89ba8ff6429b6bcb0851, regionState=CLOSED 2024-11-27T07:24:51,098 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure 18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 because future has completed 2024-11-27T07:24:51,100 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-27T07:24:51,100 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseRegionProcedure 18c429ab939f89ba8ff6429b6bcb0851, server=289954c9202d,42117,1732692259671 in 1.5960 sec 2024-11-27T07:24:51,102 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-27T07:24:51,102 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=18c429ab939f89ba8ff6429b6bcb0851, UNASSIGN in 1.6020 sec 2024-11-27T07:24:51,104 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-27T07:24:51,104 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6080 sec 2024-11-27T07:24:51,105 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732692291105"}]},"ts":"1732692291105"} 2024-11-27T07:24:51,106 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-27T07:24:51,106 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-27T07:24:51,108 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6220 sec 2024-11-27T07:24:51,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-27T07:24:51,641 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-27T07:24:51,647 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-27T07:24:51,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T07:24:51,656 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T07:24:51,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-27T07:24:51,658 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=33, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T07:24:51,663 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851 2024-11-27T07:24:51,679 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A, FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B, FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C, FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/recovered.edits] 2024-11-27T07:24:51,683 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/7bec5ea4952d45518f6543c631541882 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/7bec5ea4952d45518f6543c631541882 2024-11-27T07:24:51,685 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/7eecfe06648841ff817b83bf3a92a3f3 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/A/7eecfe06648841ff817b83bf3a92a3f3 2024-11-27T07:24:51,688 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/2f585c6cd189480d8fccf1e653999413 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/2f585c6cd189480d8fccf1e653999413 2024-11-27T07:24:51,690 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/f113f39bfd294ff281b05ba799fa15b6 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/B/f113f39bfd294ff281b05ba799fa15b6 2024-11-27T07:24:51,692 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/8a8a9eb51d384eaf8346de2a7d7bed4e to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/8a8a9eb51d384eaf8346de2a7d7bed4e 2024-11-27T07:24:51,694 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/ffa0900560fe4b3b9de9ab6da1292d5c to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/C/ffa0900560fe4b3b9de9ab6da1292d5c 2024-11-27T07:24:51,697 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/recovered.edits/451.seqid to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851/recovered.edits/451.seqid 2024-11-27T07:24:51,698 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/18c429ab939f89ba8ff6429b6bcb0851 2024-11-27T07:24:51,698 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-27T07:24:51,705 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=33, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T07:24:51,711 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-27T07:24:51,718 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-27T07:24:51,754 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-27T07:24:51,756 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=33, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T07:24:51,756 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-27T07:24:51,756 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732692291756"}]},"ts":"9223372036854775807"} 2024-11-27T07:24:51,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-27T07:24:51,769 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-27T07:24:51,769 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 18c429ab939f89ba8ff6429b6bcb0851, NAME => 'TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851.', STARTKEY => '', ENDKEY => ''}] 2024-11-27T07:24:51,769 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-27T07:24:51,770 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732692291769"}]},"ts":"9223372036854775807"} 2024-11-27T07:24:51,773 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-27T07:24:51,774 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=33, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T07:24:51,776 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 125 msec 2024-11-27T07:24:51,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-27T07:24:51,980 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-27T07:24:51,980 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-27T07:24:51,991 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=239 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1984458721_22 at /127.0.0.1:50458 [Waiting for operation #260] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-730273825_22 at /127.0.0.1:45052 [Waiting for operation #275] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS:0;289954c9202d:42117-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/289954c9202d:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/289954c9202d:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-730273825_22 at /127.0.0.1:44460 [Waiting for operation #88] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=507 (was 414) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4785 (was 5822) 2024-11-27T07:24:52,001 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=239, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=507, ProcessCount=11, AvailableMemoryMB=4784 2024-11-27T07:24:52,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-27T07:24:52,003 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T07:24:52,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-27T07:24:52,005 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T07:24:52,006 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:52,006 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 34 2024-11-27T07:24:52,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-27T07:24:52,007 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T07:24:52,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741936_1112 (size=959) 2024-11-27T07:24:52,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-27T07:24:52,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-27T07:24:52,416 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4 2024-11-27T07:24:52,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741937_1113 (size=53) 2024-11-27T07:24:52,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-27T07:24:52,828 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T07:24:52,828 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 225833644b1bebd5fa53ae861d7c7070, disabling compactions & flushes 2024-11-27T07:24:52,828 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:24:52,828 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:24:52,828 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. after waiting 0 ms 2024-11-27T07:24:52,828 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:24:52,828 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:24:52,828 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 225833644b1bebd5fa53ae861d7c7070: Waiting for close lock at 1732692292828Disabling compacts and flushes for region at 1732692292828Disabling writes for close at 1732692292828Writing region close event to WAL at 1732692292828Closed at 1732692292828 2024-11-27T07:24:52,829 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T07:24:52,830 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732692292829"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732692292829"}]},"ts":"1732692292829"} 2024-11-27T07:24:52,831 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T07:24:52,832 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T07:24:52,832 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732692292832"}]},"ts":"1732692292832"} 2024-11-27T07:24:52,833 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-27T07:24:52,834 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=225833644b1bebd5fa53ae861d7c7070, ASSIGN}] 2024-11-27T07:24:52,835 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=225833644b1bebd5fa53ae861d7c7070, ASSIGN 2024-11-27T07:24:52,835 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=225833644b1bebd5fa53ae861d7c7070, ASSIGN; state=OFFLINE, location=289954c9202d,42117,1732692259671; forceNewPlan=false, retain=false 2024-11-27T07:24:52,987 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=225833644b1bebd5fa53ae861d7c7070, regionState=OPENING, regionLocation=289954c9202d,42117,1732692259671 2024-11-27T07:24:52,992 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=225833644b1bebd5fa53ae861d7c7070, ASSIGN because future has completed 2024-11-27T07:24:52,993 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671}] 2024-11-27T07:24:53,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-27T07:24:53,157 INFO [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:24:53,157 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} 2024-11-27T07:24:53,158 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:24:53,158 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T07:24:53,158 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for 225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:24:53,158 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for 225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:24:53,160 INFO [StoreOpener-225833644b1bebd5fa53ae861d7c7070-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:24:53,162 INFO [StoreOpener-225833644b1bebd5fa53ae861d7c7070-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 225833644b1bebd5fa53ae861d7c7070 columnFamilyName A 2024-11-27T07:24:53,162 DEBUG [StoreOpener-225833644b1bebd5fa53ae861d7c7070-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:53,162 INFO [StoreOpener-225833644b1bebd5fa53ae861d7c7070-1 {}] regionserver.HStore(327): Store=225833644b1bebd5fa53ae861d7c7070/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T07:24:53,162 INFO [StoreOpener-225833644b1bebd5fa53ae861d7c7070-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:24:53,164 INFO [StoreOpener-225833644b1bebd5fa53ae861d7c7070-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 225833644b1bebd5fa53ae861d7c7070 columnFamilyName B 2024-11-27T07:24:53,164 DEBUG [StoreOpener-225833644b1bebd5fa53ae861d7c7070-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:53,164 INFO [StoreOpener-225833644b1bebd5fa53ae861d7c7070-1 {}] regionserver.HStore(327): Store=225833644b1bebd5fa53ae861d7c7070/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T07:24:53,164 INFO [StoreOpener-225833644b1bebd5fa53ae861d7c7070-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:24:53,165 INFO [StoreOpener-225833644b1bebd5fa53ae861d7c7070-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 225833644b1bebd5fa53ae861d7c7070 columnFamilyName C 2024-11-27T07:24:53,166 DEBUG [StoreOpener-225833644b1bebd5fa53ae861d7c7070-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:53,166 INFO [StoreOpener-225833644b1bebd5fa53ae861d7c7070-1 {}] regionserver.HStore(327): Store=225833644b1bebd5fa53ae861d7c7070/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T07:24:53,166 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for 225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:24:53,167 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:24:53,167 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:24:53,168 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for 225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:24:53,168 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for 225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:24:53,169 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T07:24:53,170 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for 225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:24:53,173 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T07:24:53,173 INFO [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened 225833644b1bebd5fa53ae861d7c7070; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72492287, jitterRate=0.08021925389766693}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T07:24:53,173 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:24:53,174 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for 225833644b1bebd5fa53ae861d7c7070: Running coprocessor pre-open hook at 1732692293158Writing region info on filesystem at 1732692293158Initializing all the Stores at 1732692293159 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732692293159Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732692293159Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732692293159Cleaning up temporary data from old regions at 1732692293168 (+9 ms)Running coprocessor post-open hooks at 1732692293173 (+5 ms)Region opened successfully at 1732692293174 (+1 ms) 2024-11-27T07:24:53,175 INFO [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., pid=36, masterSystemTime=1732692293147 2024-11-27T07:24:53,177 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:24:53,177 INFO [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:24:53,178 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=225833644b1bebd5fa53ae861d7c7070, regionState=OPEN, openSeqNum=2, regionLocation=289954c9202d,42117,1732692259671 2024-11-27T07:24:53,179 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 because future has completed 2024-11-27T07:24:53,183 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-27T07:24:53,183 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure 225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 in 188 msec 2024-11-27T07:24:53,185 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-27T07:24:53,185 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=225833644b1bebd5fa53ae861d7c7070, ASSIGN in 349 msec 2024-11-27T07:24:53,186 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T07:24:53,186 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732692293186"}]},"ts":"1732692293186"} 2024-11-27T07:24:53,188 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-27T07:24:53,189 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T07:24:53,191 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1860 sec 2024-11-27T07:24:54,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-27T07:24:54,151 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-27T07:24:54,154 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@328db2e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:24:54,154 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 289954c9202d,43057,-1 for getting cluster id 2024-11-27T07:24:54,155 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T07:24:54,159 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '09f43d3f-c175-4713-a9ce-038f5695b74b' 2024-11-27T07:24:54,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T07:24:54,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "09f43d3f-c175-4713-a9ce-038f5695b74b" 2024-11-27T07:24:54,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1896484f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:24:54,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [289954c9202d,43057,-1] 2024-11-27T07:24:54,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T07:24:54,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:24:54,161 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59188, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T07:24:54,161 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fd695c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:24:54,162 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T07:24:54,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=289954c9202d,42117,1732692259671, seqNum=-1] 2024-11-27T07:24:54,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T07:24:54,164 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36286, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T07:24:54,165 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-27T07:24:54,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 289954c9202d,43057,1732692258941 2024-11-27T07:24:54,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7c72d130 2024-11-27T07:24:54,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-27T07:24:54,167 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59194, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-27T07:24:54,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-27T07:24:54,172 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T07:24:54,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] procedure2.ProcedureExecutor(1139): Stored pid=37, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-27T07:24:54,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741938_1114 (size=995) 2024-11-27T07:24:54,596 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-27T07:24:54,597 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-27T07:24:54,602 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-27T07:24:54,608 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=225833644b1bebd5fa53ae861d7c7070, REOPEN/MOVE}] 2024-11-27T07:24:54,609 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=225833644b1bebd5fa53ae861d7c7070, REOPEN/MOVE 2024-11-27T07:24:54,609 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=225833644b1bebd5fa53ae861d7c7070, regionState=CLOSING, regionLocation=289954c9202d,42117,1732692259671 2024-11-27T07:24:54,611 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=225833644b1bebd5fa53ae861d7c7070, REOPEN/MOVE because future has completed 2024-11-27T07:24:54,611 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T07:24:54,611 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure 225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671}] 2024-11-27T07:24:54,764 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(122): Close 225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:24:54,764 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T07:24:54,764 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1722): Closing 225833644b1bebd5fa53ae861d7c7070, disabling compactions & flushes 2024-11-27T07:24:54,764 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:24:54,764 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:24:54,764 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. after waiting 0 ms 2024-11-27T07:24:54,764 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:24:54,770 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-27T07:24:54,771 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:24:54,771 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1676): Region close journal for 225833644b1bebd5fa53ae861d7c7070: Waiting for close lock at 1732692294764Running coprocessor pre-close hooks at 1732692294764Disabling compacts and flushes for region at 1732692294764Disabling writes for close at 1732692294764Writing region close event to WAL at 1732692294765 (+1 ms)Running coprocessor post-close hooks at 1732692294771 (+6 ms)Closed at 1732692294771 2024-11-27T07:24:54,771 WARN [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegionServer(3299): Not adding moved region record: 225833644b1bebd5fa53ae861d7c7070 to self. 2024-11-27T07:24:54,775 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(157): Closed 225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:24:54,775 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=225833644b1bebd5fa53ae861d7c7070, regionState=CLOSED 2024-11-27T07:24:54,778 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure 225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 because future has completed 2024-11-27T07:24:54,784 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-11-27T07:24:54,784 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; CloseRegionProcedure 225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 in 169 msec 2024-11-27T07:24:54,785 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=225833644b1bebd5fa53ae861d7c7070, REOPEN/MOVE; state=CLOSED, location=289954c9202d,42117,1732692259671; forceNewPlan=false, retain=true 2024-11-27T07:24:54,937 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=225833644b1bebd5fa53ae861d7c7070, regionState=OPENING, regionLocation=289954c9202d,42117,1732692259671 2024-11-27T07:24:54,942 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=225833644b1bebd5fa53ae861d7c7070, REOPEN/MOVE because future has completed 2024-11-27T07:24:54,943 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure 225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671}] 2024-11-27T07:24:55,104 INFO [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:24:55,105 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} 2024-11-27T07:24:55,105 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:24:55,106 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T07:24:55,106 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for 225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:24:55,106 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for 225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:24:55,112 INFO [StoreOpener-225833644b1bebd5fa53ae861d7c7070-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:24:55,119 INFO [StoreOpener-225833644b1bebd5fa53ae861d7c7070-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 225833644b1bebd5fa53ae861d7c7070 columnFamilyName A 2024-11-27T07:24:55,120 DEBUG [StoreOpener-225833644b1bebd5fa53ae861d7c7070-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:55,121 INFO [StoreOpener-225833644b1bebd5fa53ae861d7c7070-1 {}] regionserver.HStore(327): Store=225833644b1bebd5fa53ae861d7c7070/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T07:24:55,121 INFO [StoreOpener-225833644b1bebd5fa53ae861d7c7070-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:24:55,122 INFO [StoreOpener-225833644b1bebd5fa53ae861d7c7070-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 225833644b1bebd5fa53ae861d7c7070 columnFamilyName B 2024-11-27T07:24:55,122 DEBUG [StoreOpener-225833644b1bebd5fa53ae861d7c7070-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:55,123 INFO [StoreOpener-225833644b1bebd5fa53ae861d7c7070-1 {}] regionserver.HStore(327): Store=225833644b1bebd5fa53ae861d7c7070/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T07:24:55,123 INFO [StoreOpener-225833644b1bebd5fa53ae861d7c7070-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:24:55,123 INFO [StoreOpener-225833644b1bebd5fa53ae861d7c7070-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 225833644b1bebd5fa53ae861d7c7070 columnFamilyName C 2024-11-27T07:24:55,124 DEBUG [StoreOpener-225833644b1bebd5fa53ae861d7c7070-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:55,124 INFO [StoreOpener-225833644b1bebd5fa53ae861d7c7070-1 {}] regionserver.HStore(327): Store=225833644b1bebd5fa53ae861d7c7070/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T07:24:55,124 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for 225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:24:55,125 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:24:55,126 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:24:55,127 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for 225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:24:55,127 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for 225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:24:55,127 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T07:24:55,129 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for 225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:24:55,129 INFO [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened 225833644b1bebd5fa53ae861d7c7070; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67160556, jitterRate=7.70270824432373E-4}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T07:24:55,130 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:24:55,132 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for 225833644b1bebd5fa53ae861d7c7070: Running coprocessor pre-open hook at 1732692295106Writing region info on filesystem at 1732692295106Initializing all the Stores at 1732692295109 (+3 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732692295109Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732692295109Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732692295109Cleaning up temporary data from old regions at 1732692295127 (+18 ms)Running coprocessor post-open hooks at 1732692295130 (+3 ms)Region opened successfully at 1732692295132 (+2 ms) 2024-11-27T07:24:55,133 INFO [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., pid=41, masterSystemTime=1732692295097 2024-11-27T07:24:55,135 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:24:55,135 INFO [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:24:55,135 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=225833644b1bebd5fa53ae861d7c7070, regionState=OPEN, openSeqNum=5, regionLocation=289954c9202d,42117,1732692259671 2024-11-27T07:24:55,137 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure 225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 because future has completed 2024-11-27T07:24:55,140 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=39 2024-11-27T07:24:55,140 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=39, state=SUCCESS, hasLock=false; OpenRegionProcedure 225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 in 195 msec 2024-11-27T07:24:55,142 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-27T07:24:55,142 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=225833644b1bebd5fa53ae861d7c7070, REOPEN/MOVE in 532 msec 2024-11-27T07:24:55,145 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-27T07:24:55,145 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 541 msec 2024-11-27T07:24:55,148 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 971 msec 2024-11-27T07:24:55,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=37 2024-11-27T07:24:55,150 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-27T07:24:55,151 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@263f8ca5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:24:55,151 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 289954c9202d,43057,-1 for getting cluster id 2024-11-27T07:24:55,151 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T07:24:55,152 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '09f43d3f-c175-4713-a9ce-038f5695b74b' 2024-11-27T07:24:55,152 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T07:24:55,152 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "09f43d3f-c175-4713-a9ce-038f5695b74b" 2024-11-27T07:24:55,152 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78b0d98e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:24:55,152 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [289954c9202d,43057,-1] 2024-11-27T07:24:55,152 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T07:24:55,153 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:24:55,153 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59212, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T07:24:55,154 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@554d2b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:24:55,155 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65b7b5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:24:55,155 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 289954c9202d,43057,-1 for getting cluster id 2024-11-27T07:24:55,155 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T07:24:55,156 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '09f43d3f-c175-4713-a9ce-038f5695b74b' 2024-11-27T07:24:55,156 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T07:24:55,156 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "09f43d3f-c175-4713-a9ce-038f5695b74b" 2024-11-27T07:24:55,156 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1389b68a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:24:55,157 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [289954c9202d,43057,-1] 2024-11-27T07:24:55,157 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T07:24:55,157 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:24:55,158 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59234, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T07:24:55,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5147dd57, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:24:55,159 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a1845b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:24:55,159 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 289954c9202d,43057,-1 for getting cluster id 2024-11-27T07:24:55,159 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T07:24:55,160 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '09f43d3f-c175-4713-a9ce-038f5695b74b' 2024-11-27T07:24:55,160 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T07:24:55,161 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "09f43d3f-c175-4713-a9ce-038f5695b74b" 2024-11-27T07:24:55,161 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5da4069, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:24:55,161 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [289954c9202d,43057,-1] 2024-11-27T07:24:55,161 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T07:24:55,161 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:24:55,162 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59246, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T07:24:55,162 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ae946ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:24:55,163 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b0484de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:24:55,163 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 289954c9202d,43057,-1 for getting cluster id 2024-11-27T07:24:55,163 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T07:24:55,164 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '09f43d3f-c175-4713-a9ce-038f5695b74b' 2024-11-27T07:24:55,164 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T07:24:55,164 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "09f43d3f-c175-4713-a9ce-038f5695b74b" 2024-11-27T07:24:55,165 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78c03057, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:24:55,165 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [289954c9202d,43057,-1] 2024-11-27T07:24:55,165 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T07:24:55,165 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:24:55,166 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59266, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T07:24:55,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71fe4469, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:24:55,167 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@349085e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:24:55,167 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 289954c9202d,43057,-1 for getting cluster id 2024-11-27T07:24:55,167 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T07:24:55,168 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '09f43d3f-c175-4713-a9ce-038f5695b74b' 2024-11-27T07:24:55,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T07:24:55,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "09f43d3f-c175-4713-a9ce-038f5695b74b" 2024-11-27T07:24:55,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54deb879, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:24:55,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [289954c9202d,43057,-1] 2024-11-27T07:24:55,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T07:24:55,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:24:55,170 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59292, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T07:24:55,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c4a1ff3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:24:55,171 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26dcfede, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:24:55,171 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 289954c9202d,43057,-1 for getting cluster id 2024-11-27T07:24:55,171 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T07:24:55,172 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '09f43d3f-c175-4713-a9ce-038f5695b74b' 2024-11-27T07:24:55,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T07:24:55,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "09f43d3f-c175-4713-a9ce-038f5695b74b" 2024-11-27T07:24:55,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66bbeb26, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:24:55,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [289954c9202d,43057,-1] 2024-11-27T07:24:55,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T07:24:55,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:24:55,174 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59310, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T07:24:55,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31989b5a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:24:55,175 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d21c225, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:24:55,175 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 289954c9202d,43057,-1 for getting cluster id 2024-11-27T07:24:55,175 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T07:24:55,176 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '09f43d3f-c175-4713-a9ce-038f5695b74b' 2024-11-27T07:24:55,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T07:24:55,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "09f43d3f-c175-4713-a9ce-038f5695b74b" 2024-11-27T07:24:55,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42285933, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:24:55,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [289954c9202d,43057,-1] 2024-11-27T07:24:55,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T07:24:55,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:24:55,177 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59316, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T07:24:55,178 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@580c75eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:24:55,179 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a92e196, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:24:55,179 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 289954c9202d,43057,-1 for getting cluster id 2024-11-27T07:24:55,179 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T07:24:55,180 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '09f43d3f-c175-4713-a9ce-038f5695b74b' 2024-11-27T07:24:55,180 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T07:24:55,180 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "09f43d3f-c175-4713-a9ce-038f5695b74b" 2024-11-27T07:24:55,180 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e90d2cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:24:55,180 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [289954c9202d,43057,-1] 2024-11-27T07:24:55,180 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T07:24:55,180 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:24:55,181 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59332, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T07:24:55,182 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1701504c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:24:55,183 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3806f988, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:24:55,183 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 289954c9202d,43057,-1 for getting cluster id 2024-11-27T07:24:55,183 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T07:24:55,184 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '09f43d3f-c175-4713-a9ce-038f5695b74b' 2024-11-27T07:24:55,184 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T07:24:55,184 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "09f43d3f-c175-4713-a9ce-038f5695b74b" 2024-11-27T07:24:55,184 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20cdd4e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:24:55,184 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [289954c9202d,43057,-1] 2024-11-27T07:24:55,184 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T07:24:55,184 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:24:55,185 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59346, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T07:24:55,186 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a87beba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:24:55,187 DEBUG [Thread-593 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T07:24:55,187 DEBUG [Thread-589 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T07:24:55,187 DEBUG [Thread-587 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T07:24:55,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=289954c9202d,42117,1732692259671, seqNum=-1] 2024-11-27T07:24:55,188 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=289954c9202d,42117,1732692259671, seqNum=-1] 2024-11-27T07:24:55,188 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T07:24:55,188 DEBUG [Thread-602 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T07:24:55,188 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T07:24:55,188 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=289954c9202d,42117,1732692259671, seqNum=-1] 2024-11-27T07:24:55,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T07:24:55,189 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=289954c9202d,42117,1732692259671, seqNum=-1] 2024-11-27T07:24:55,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-27T07:24:55,190 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T07:24:55,190 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36298, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T07:24:55,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-27T07:24:55,191 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36302, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T07:24:55,191 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T07:24:55,191 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T07:24:55,192 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36312, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T07:24:55,192 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36314, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T07:24:55,193 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T07:24:55,193 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T07:24:55,195 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5] 2024-11-27T07:24:55,196 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5] 2024-11-27T07:24:55,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5] 2024-11-27T07:24:55,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5] 2024-11-27T07:24:55,199 DEBUG [Thread-600 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T07:24:55,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=289954c9202d,42117,1732692259671, seqNum=-1] 2024-11-27T07:24:55,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T07:24:55,201 DEBUG [Thread-595 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T07:24:55,202 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36336, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T07:24:55,202 DEBUG [Thread-591 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T07:24:55,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=289954c9202d,42117,1732692259671, seqNum=-1] 2024-11-27T07:24:55,203 DEBUG [Thread-598 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T07:24:55,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=289954c9202d,42117,1732692259671, seqNum=-1] 2024-11-27T07:24:55,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T07:24:55,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T07:24:55,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=289954c9202d,42117,1732692259671, seqNum=-1] 2024-11-27T07:24:55,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T07:24:55,204 DEBUG [Thread-604 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T07:24:55,205 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36352, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T07:24:55,205 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5] 2024-11-27T07:24:55,206 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=289954c9202d,42117,1732692259671, seqNum=-1] 2024-11-27T07:24:55,206 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36338, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T07:24:55,206 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T07:24:55,207 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36364, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T07:24:55,208 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36374, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T07:24:55,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5] 2024-11-27T07:24:55,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5] 2024-11-27T07:24:55,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:24:55,209 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T07:24:55,210 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 225833644b1bebd5fa53ae861d7c7070 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-27T07:24:55,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5] 2024-11-27T07:24:55,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5] 2024-11-27T07:24:55,296 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411271e119778b7544994a69a28a716e54b69_225833644b1bebd5fa53ae861d7c7070 is 50, key is test_row_0/A:col10/1732692295205/Put/seqid=0 2024-11-27T07:24:55,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-27T07:24:55,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:55,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36298 deadline: 1732692355304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:55,316 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:55,316 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:55,316 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:55,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:55,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36312 deadline: 1732692355305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:55,317 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:55,317 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:55,317 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:55,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:55,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36352 deadline: 1732692355306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:55,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:55,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:55,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:55,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:55,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36314 deadline: 1732692355315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:55,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:55,323 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:55,323 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:55,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:55,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36338 deadline: 1732692355316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:55,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:55,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:55,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:55,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741939_1115 (size=12154) 2024-11-27T07:24:55,329 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:55,337 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411271e119778b7544994a69a28a716e54b69_225833644b1bebd5fa53ae861d7c7070 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411271e119778b7544994a69a28a716e54b69_225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:24:55,338 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/f2f55e9ff30a4beaa03dd8e1018f4660, store: [table=TestAcidGuarantees family=A region=225833644b1bebd5fa53ae861d7c7070] 2024-11-27T07:24:55,346 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-27T07:24:55,346 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/f2f55e9ff30a4beaa03dd8e1018f4660 is 175, key is test_row_0/A:col10/1732692295205/Put/seqid=0 2024-11-27T07:24:55,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:24:55,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. as already flushing 2024-11-27T07:24:55,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:24:55,346 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:55,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:55,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:55,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741940_1116 (size=30955) 2024-11-27T07:24:55,368 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/f2f55e9ff30a4beaa03dd8e1018f4660 2024-11-27T07:24:55,395 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/30b91302f41b4211b9d13219d2384e57 is 50, key is test_row_0/B:col10/1732692295205/Put/seqid=0 2024-11-27T07:24:55,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741941_1117 (size=12001) 2024-11-27T07:24:55,409 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/30b91302f41b4211b9d13219d2384e57 2024-11-27T07:24:55,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:55,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36298 deadline: 1732692355419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:55,425 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:55,425 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:55,425 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:55,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:55,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36314 deadline: 1732692355430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:55,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:55,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:55,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:55,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:55,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36338 deadline: 1732692355430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:55,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:55,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36312 deadline: 1732692355431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:55,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:55,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:55,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:55,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:55,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36352 deadline: 1732692355431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:55,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:55,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:55,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:55,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:55,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:55,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:55,448 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/50073d7d11ea45c4bdb1951d8c0608b4 is 50, key is test_row_0/C:col10/1732692295205/Put/seqid=0 2024-11-27T07:24:55,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741942_1118 (size=12001) 2024-11-27T07:24:55,472 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/50073d7d11ea45c4bdb1951d8c0608b4 2024-11-27T07:24:55,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/f2f55e9ff30a4beaa03dd8e1018f4660 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/f2f55e9ff30a4beaa03dd8e1018f4660 2024-11-27T07:24:55,491 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/f2f55e9ff30a4beaa03dd8e1018f4660, entries=150, sequenceid=16, filesize=30.2 K 2024-11-27T07:24:55,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/30b91302f41b4211b9d13219d2384e57 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/30b91302f41b4211b9d13219d2384e57 2024-11-27T07:24:55,499 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/30b91302f41b4211b9d13219d2384e57, entries=150, sequenceid=16, filesize=11.7 K 2024-11-27T07:24:55,500 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-27T07:24:55,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:24:55,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. as already flushing 2024-11-27T07:24:55,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:24:55,501 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:55,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:55,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:55,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-27T07:24:55,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/50073d7d11ea45c4bdb1951d8c0608b4 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/50073d7d11ea45c4bdb1951d8c0608b4 2024-11-27T07:24:55,529 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/50073d7d11ea45c4bdb1951d8c0608b4, entries=150, sequenceid=16, filesize=11.7 K 2024-11-27T07:24:55,530 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 225833644b1bebd5fa53ae861d7c7070 in 321ms, sequenceid=16, compaction requested=false 2024-11-27T07:24:55,531 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-27T07:24:55,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 225833644b1bebd5fa53ae861d7c7070: 2024-11-27T07:24:55,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:24:55,633 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T07:24:55,633 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 225833644b1bebd5fa53ae861d7c7070 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-27T07:24:55,649 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411272dd0879ca8fe48c4a1bf987310c3819f_225833644b1bebd5fa53ae861d7c7070 is 50, key is test_row_0/A:col10/1732692295298/Put/seqid=0 2024-11-27T07:24:55,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-27T07:24:55,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:24:55,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. as already flushing 2024-11-27T07:24:55,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:24:55,656 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:55,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:55,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:55,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741943_1119 (size=14594) 2024-11-27T07:24:55,671 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:55,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:55,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36352 deadline: 1732692355654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:55,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:55,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:55,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:55,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:55,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36298 deadline: 1732692355658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:55,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:55,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:55,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:55,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:55,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36312 deadline: 1732692355660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:55,677 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:55,677 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:55,677 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:55,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:55,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36338 deadline: 1732692355667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:55,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:55,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36314 deadline: 1732692355676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:55,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:55,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:55,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:55,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:55,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:55,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:55,688 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411272dd0879ca8fe48c4a1bf987310c3819f_225833644b1bebd5fa53ae861d7c7070 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411272dd0879ca8fe48c4a1bf987310c3819f_225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:24:55,690 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/2e718dcb21254a48af41e0ae08b92261, store: [table=TestAcidGuarantees family=A region=225833644b1bebd5fa53ae861d7c7070] 2024-11-27T07:24:55,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/2e718dcb21254a48af41e0ae08b92261 is 175, key is test_row_0/A:col10/1732692295298/Put/seqid=0 2024-11-27T07:24:55,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741944_1120 (size=39549) 2024-11-27T07:24:55,702 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/2e718dcb21254a48af41e0ae08b92261 2024-11-27T07:24:55,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/09c2adf80b6c483a97d5d5c288a2b975 is 50, key is test_row_0/B:col10/1732692295298/Put/seqid=0 2024-11-27T07:24:55,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741945_1121 (size=12001) 2024-11-27T07:24:55,738 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/09c2adf80b6c483a97d5d5c288a2b975 2024-11-27T07:24:55,750 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/96c1c4adf48e454780131cebf161acbf is 50, key is test_row_0/C:col10/1732692295298/Put/seqid=0 2024-11-27T07:24:55,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741946_1122 (size=12001) 2024-11-27T07:24:55,767 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/96c1c4adf48e454780131cebf161acbf 2024-11-27T07:24:55,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/2e718dcb21254a48af41e0ae08b92261 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/2e718dcb21254a48af41e0ae08b92261 2024-11-27T07:24:55,781 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/2e718dcb21254a48af41e0ae08b92261, entries=200, sequenceid=41, filesize=38.6 K 2024-11-27T07:24:55,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/09c2adf80b6c483a97d5d5c288a2b975 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/09c2adf80b6c483a97d5d5c288a2b975 2024-11-27T07:24:55,787 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/09c2adf80b6c483a97d5d5c288a2b975, entries=150, sequenceid=41, filesize=11.7 K 2024-11-27T07:24:55,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/96c1c4adf48e454780131cebf161acbf as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/96c1c4adf48e454780131cebf161acbf 2024-11-27T07:24:55,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:55,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36352 deadline: 1732692355779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:55,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:55,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:55,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:55,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:55,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36312 deadline: 1732692355779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:55,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:55,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:55,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:55,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:55,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36298 deadline: 1732692355779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:55,792 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:55,792 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:55,792 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:55,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:55,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36338 deadline: 1732692355789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:55,797 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/96c1c4adf48e454780131cebf161acbf, entries=150, sequenceid=41, filesize=11.7 K 2024-11-27T07:24:55,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:55,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:55,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36314 deadline: 1732692355789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:55,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:55,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:55,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:55,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:55,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:55,798 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 225833644b1bebd5fa53ae861d7c7070 in 165ms, sequenceid=41, compaction requested=false 2024-11-27T07:24:55,798 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 225833644b1bebd5fa53ae861d7c7070: 2024-11-27T07:24:55,810 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-27T07:24:55,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:24:55,810 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 225833644b1bebd5fa53ae861d7c7070 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-27T07:24:55,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-27T07:24:55,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112734c0f711b43b4310a964dd26288e6b3f_225833644b1bebd5fa53ae861d7c7070 is 50, key is test_row_0/A:col10/1732692295660/Put/seqid=0 2024-11-27T07:24:55,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741947_1123 (size=12154) 2024-11-27T07:24:55,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:55,855 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112734c0f711b43b4310a964dd26288e6b3f_225833644b1bebd5fa53ae861d7c7070 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112734c0f711b43b4310a964dd26288e6b3f_225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:24:55,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/29ac21a055c0427f88779f2b67b83ddb, store: [table=TestAcidGuarantees family=A region=225833644b1bebd5fa53ae861d7c7070] 2024-11-27T07:24:55,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/29ac21a055c0427f88779f2b67b83ddb is 175, key is test_row_0/A:col10/1732692295660/Put/seqid=0 2024-11-27T07:24:55,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741948_1124 (size=30955) 2024-11-27T07:24:55,888 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/29ac21a055c0427f88779f2b67b83ddb 2024-11-27T07:24:55,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/d7c999b8003748c3b983cb52d30400ee is 50, key is test_row_0/B:col10/1732692295660/Put/seqid=0 2024-11-27T07:24:55,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741949_1125 (size=12001) 2024-11-27T07:24:56,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:24:56,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. as already flushing 2024-11-27T07:24:56,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:56,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36312 deadline: 1732692356049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:56,061 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:56,061 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:56,061 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:56,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:56,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36298 deadline: 1732692356050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:56,062 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:56,062 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:56,062 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:56,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:56,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36314 deadline: 1732692356051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:56,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:56,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:56,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:56,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:56,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36352 deadline: 1732692356055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:56,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:56,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:56,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:56,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:56,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36338 deadline: 1732692356059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:56,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:56,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:56,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:56,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:56,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36312 deadline: 1732692356169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:56,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:56,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36314 deadline: 1732692356169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:56,171 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:56,171 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:56,171 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:56,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:56,171 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:56,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36298 deadline: 1732692356169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:56,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:56,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:56,172 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:56,172 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:56,172 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:56,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:56,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36338 deadline: 1732692356170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:56,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:56,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:56,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:56,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:56,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36352 deadline: 1732692356170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:56,173 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:56,173 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:56,173 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:56,313 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/d7c999b8003748c3b983cb52d30400ee 2024-11-27T07:24:56,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/3795192dd17d42aeba63ad4baf94ec79 is 50, key is test_row_0/C:col10/1732692295660/Put/seqid=0 2024-11-27T07:24:56,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-27T07:24:56,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741950_1126 (size=12001) 2024-11-27T07:24:56,338 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/3795192dd17d42aeba63ad4baf94ec79 2024-11-27T07:24:56,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/29ac21a055c0427f88779f2b67b83ddb as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/29ac21a055c0427f88779f2b67b83ddb 2024-11-27T07:24:56,353 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/29ac21a055c0427f88779f2b67b83ddb, entries=150, sequenceid=52, filesize=30.2 K 2024-11-27T07:24:56,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/d7c999b8003748c3b983cb52d30400ee as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/d7c999b8003748c3b983cb52d30400ee 2024-11-27T07:24:56,364 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/d7c999b8003748c3b983cb52d30400ee, entries=150, sequenceid=52, filesize=11.7 K 2024-11-27T07:24:56,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/3795192dd17d42aeba63ad4baf94ec79 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/3795192dd17d42aeba63ad4baf94ec79 2024-11-27T07:24:56,375 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/3795192dd17d42aeba63ad4baf94ec79, entries=150, sequenceid=52, filesize=11.7 K 2024-11-27T07:24:56,377 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 225833644b1bebd5fa53ae861d7c7070 in 567ms, sequenceid=52, compaction requested=true 2024-11-27T07:24:56,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 225833644b1bebd5fa53ae861d7c7070: 2024-11-27T07:24:56,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:24:56,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-27T07:24:56,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-27T07:24:56,383 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-27T07:24:56,383 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1860 sec 2024-11-27T07:24:56,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:24:56,386 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T07:24:56,386 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.1960 sec 2024-11-27T07:24:56,386 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 225833644b1bebd5fa53ae861d7c7070 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-27T07:24:56,400 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411277e5351021b59446ab00728b7dbca8eb5_225833644b1bebd5fa53ae861d7c7070 is 50, key is test_row_0/A:col10/1732692296384/Put/seqid=0 2024-11-27T07:24:56,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:56,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36352 deadline: 1732692356389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:56,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:56,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:56,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:56,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:56,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:56,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36298 deadline: 1732692356398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:56,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36338 deadline: 1732692356394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:56,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:56,406 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:56,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:56,406 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:56,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:56,406 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:56,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:56,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:56,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36312 deadline: 1732692356401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:56,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36314 deadline: 1732692356400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:56,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:56,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:56,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:56,408 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:56,408 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:56,408 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:56,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741951_1127 (size=12154) 2024-11-27T07:24:56,415 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,422 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411277e5351021b59446ab00728b7dbca8eb5_225833644b1bebd5fa53ae861d7c7070 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411277e5351021b59446ab00728b7dbca8eb5_225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:24:56,424 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/f965528058614b93999d39c30bff5895, store: [table=TestAcidGuarantees family=A region=225833644b1bebd5fa53ae861d7c7070] 2024-11-27T07:24:56,425 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/f965528058614b93999d39c30bff5895 is 175, key is test_row_0/A:col10/1732692296384/Put/seqid=0 2024-11-27T07:24:56,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741952_1128 (size=30955) 2024-11-27T07:24:56,440 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=80, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/f965528058614b93999d39c30bff5895 2024-11-27T07:24:56,454 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/1a9802761b8c4a8caef051adff35c284 is 50, key is test_row_0/B:col10/1732692296384/Put/seqid=0 2024-11-27T07:24:56,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741953_1129 (size=12001) 2024-11-27T07:24:56,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:56,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36352 deadline: 1732692356509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:56,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:56,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:56,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36298 deadline: 1732692356510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:56,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36338 deadline: 1732692356510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:56,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:56,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:56,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:56,513 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:56,513 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:56,513 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:56,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:56,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:56,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:56,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:56,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36312 deadline: 1732692356519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:56,525 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:56,525 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:56,525 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:56,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:56,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36314 deadline: 1732692356519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:56,526 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:56,526 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:56,526 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:56,631 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T07:24:56,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:56,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36352 deadline: 1732692356720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:56,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:56,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:56,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36338 deadline: 1732692356720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:56,724 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:56,724 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:56,724 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:56,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:56,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:56,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36298 deadline: 1732692356720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:56,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:56,725 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:56,725 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:56,725 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:56,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:56,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36312 deadline: 1732692356729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:56,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:56,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:56,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36314 deadline: 1732692356729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:56,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:56,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:56,731 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:56,731 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:56,731 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:56,871 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/1a9802761b8c4a8caef051adff35c284 2024-11-27T07:24:56,884 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/35bd2437cc9e48909ac8767158bb895e is 50, key is test_row_0/C:col10/1732692296384/Put/seqid=0 2024-11-27T07:24:56,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741954_1130 (size=12001) 2024-11-27T07:24:56,899 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/35bd2437cc9e48909ac8767158bb895e 2024-11-27T07:24:56,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/f965528058614b93999d39c30bff5895 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/f965528058614b93999d39c30bff5895 2024-11-27T07:24:56,920 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/f965528058614b93999d39c30bff5895, entries=150, sequenceid=80, filesize=30.2 K 2024-11-27T07:24:56,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/1a9802761b8c4a8caef051adff35c284 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/1a9802761b8c4a8caef051adff35c284 2024-11-27T07:24:56,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,930 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/1a9802761b8c4a8caef051adff35c284, entries=150, sequenceid=80, filesize=11.7 K 2024-11-27T07:24:56,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/35bd2437cc9e48909ac8767158bb895e as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/35bd2437cc9e48909ac8767158bb895e 2024-11-27T07:24:56,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,939 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/35bd2437cc9e48909ac8767158bb895e, entries=150, sequenceid=80, filesize=11.7 K 2024-11-27T07:24:56,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,941 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 225833644b1bebd5fa53ae861d7c7070 in 555ms, sequenceid=80, compaction requested=true 2024-11-27T07:24:56,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 225833644b1bebd5fa53ae861d7c7070: 2024-11-27T07:24:56,942 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T07:24:56,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 225833644b1bebd5fa53ae861d7c7070:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T07:24:56,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:24:56,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 225833644b1bebd5fa53ae861d7c7070:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T07:24:56,943 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T07:24:56,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:24:56,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 225833644b1bebd5fa53ae861d7c7070:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T07:24:56,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:24:56,944 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132414 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T07:24:56,944 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1541): 225833644b1bebd5fa53ae861d7c7070/A is initiating minor compaction (all files) 2024-11-27T07:24:56,944 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 225833644b1bebd5fa53ae861d7c7070/A in TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:24:56,944 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/f2f55e9ff30a4beaa03dd8e1018f4660, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/2e718dcb21254a48af41e0ae08b92261, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/29ac21a055c0427f88779f2b67b83ddb, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/f965528058614b93999d39c30bff5895] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp, totalSize=129.3 K 2024-11-27T07:24:56,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,944 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:24:56,944 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. files: [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/f2f55e9ff30a4beaa03dd8e1018f4660, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/2e718dcb21254a48af41e0ae08b92261, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/29ac21a055c0427f88779f2b67b83ddb, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/f965528058614b93999d39c30bff5895] 2024-11-27T07:24:56,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,945 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2f55e9ff30a4beaa03dd8e1018f4660, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732692295205 2024-11-27T07:24:56,946 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T07:24:56,946 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 225833644b1bebd5fa53ae861d7c7070/B is initiating minor compaction (all files) 2024-11-27T07:24:56,946 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 225833644b1bebd5fa53ae861d7c7070/B in TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:24:56,946 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/30b91302f41b4211b9d13219d2384e57, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/09c2adf80b6c483a97d5d5c288a2b975, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/d7c999b8003748c3b983cb52d30400ee, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/1a9802761b8c4a8caef051adff35c284] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp, totalSize=46.9 K 2024-11-27T07:24:56,946 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e718dcb21254a48af41e0ae08b92261, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732692295298 2024-11-27T07:24:56,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,947 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 30b91302f41b4211b9d13219d2384e57, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732692295205 2024-11-27T07:24:56,948 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29ac21a055c0427f88779f2b67b83ddb, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732692295657 2024-11-27T07:24:56,948 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 09c2adf80b6c483a97d5d5c288a2b975, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732692295298 2024-11-27T07:24:56,948 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting f965528058614b93999d39c30bff5895, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732692296382 2024-11-27T07:24:56,948 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting d7c999b8003748c3b983cb52d30400ee, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732692295657 2024-11-27T07:24:56,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,949 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a9802761b8c4a8caef051adff35c284, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732692296382 2024-11-27T07:24:56,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,964 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=225833644b1bebd5fa53ae861d7c7070] 2024-11-27T07:24:56,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,973 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112737ea0c342b044aedaeb5a74752973656_225833644b1bebd5fa53ae861d7c7070 store=[table=TestAcidGuarantees family=A region=225833644b1bebd5fa53ae861d7c7070] 2024-11-27T07:24:56,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,977 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 225833644b1bebd5fa53ae861d7c7070#B#compaction#112 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:24:56,978 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/f5fe59552cb748e983308e4283c0e01f is 50, key is test_row_0/B:col10/1732692296384/Put/seqid=0 2024-11-27T07:24:56,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,980 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112737ea0c342b044aedaeb5a74752973656_225833644b1bebd5fa53ae861d7c7070, store=[table=TestAcidGuarantees family=A region=225833644b1bebd5fa53ae861d7c7070] 2024-11-27T07:24:56,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,980 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112737ea0c342b044aedaeb5a74752973656_225833644b1bebd5fa53ae861d7c7070 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=225833644b1bebd5fa53ae861d7c7070] 2024-11-27T07:24:56,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741956_1132 (size=4469) 2024-11-27T07:24:56,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:56,999 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 225833644b1bebd5fa53ae861d7c7070#A#compaction#111 average throughput is 0.70 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:24:57,000 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/6c90e21305ac40da83d99fa62732443b is 175, key is test_row_0/A:col10/1732692296384/Put/seqid=0 2024-11-27T07:24:57,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741955_1131 (size=12139) 2024-11-27T07:24:57,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,018 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/f5fe59552cb748e983308e4283c0e01f as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/f5fe59552cb748e983308e4283c0e01f 2024-11-27T07:24:57,024 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 225833644b1bebd5fa53ae861d7c7070/B of 225833644b1bebd5fa53ae861d7c7070 into f5fe59552cb748e983308e4283c0e01f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:24:57,024 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 225833644b1bebd5fa53ae861d7c7070: 2024-11-27T07:24:57,024 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., storeName=225833644b1bebd5fa53ae861d7c7070/B, priority=12, startTime=1732692296943; duration=0sec 2024-11-27T07:24:57,024 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:24:57,024 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 225833644b1bebd5fa53ae861d7c7070:B 2024-11-27T07:24:57,024 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T07:24:57,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,026 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T07:24:57,026 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 225833644b1bebd5fa53ae861d7c7070/C is initiating minor compaction (all files) 2024-11-27T07:24:57,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,026 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 225833644b1bebd5fa53ae861d7c7070/C in TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:24:57,026 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/50073d7d11ea45c4bdb1951d8c0608b4, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/96c1c4adf48e454780131cebf161acbf, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/3795192dd17d42aeba63ad4baf94ec79, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/35bd2437cc9e48909ac8767158bb895e] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp, totalSize=46.9 K 2024-11-27T07:24:57,026 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 50073d7d11ea45c4bdb1951d8c0608b4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732692295205 2024-11-27T07:24:57,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,027 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 96c1c4adf48e454780131cebf161acbf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732692295298 2024-11-27T07:24:57,027 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 3795192dd17d42aeba63ad4baf94ec79, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732692295657 2024-11-27T07:24:57,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,027 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 35bd2437cc9e48909ac8767158bb895e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732692296382 2024-11-27T07:24:57,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741957_1133 (size=31093) 2024-11-27T07:24:57,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:24:57,044 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T07:24:57,044 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 225833644b1bebd5fa53ae861d7c7070 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-27T07:24:57,044 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 225833644b1bebd5fa53ae861d7c7070#C#compaction#113 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:24:57,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,045 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/2276c55338654b50a52106db051a31e7 is 50, key is test_row_0/C:col10/1732692296384/Put/seqid=0 2024-11-27T07:24:57,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741958_1134 (size=12139) 2024-11-27T07:24:57,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,079 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127ba42c86a2f1f485e80580a759164eed4_225833644b1bebd5fa53ae861d7c7070 is 50, key is test_row_0/A:col10/1732692296395/Put/seqid=0 2024-11-27T07:24:57,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741959_1135 (size=17034) 2024-11-27T07:24:57,100 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:57,111 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127ba42c86a2f1f485e80580a759164eed4_225833644b1bebd5fa53ae861d7c7070 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127ba42c86a2f1f485e80580a759164eed4_225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:24:57,114 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/7c61ad07bbbe4f3683ef9d7658394644, store: [table=TestAcidGuarantees family=A region=225833644b1bebd5fa53ae861d7c7070] 2024-11-27T07:24:57,115 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/7c61ad07bbbe4f3683ef9d7658394644 is 175, key is test_row_0/A:col10/1732692296395/Put/seqid=0 2024-11-27T07:24:57,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:57,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36352 deadline: 1732692357107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:57,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:57,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:57,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:57,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:57,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36298 deadline: 1732692357109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:57,118 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:57,118 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:57,118 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:57,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:57,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36314 deadline: 1732692357114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:57,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:57,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:57,122 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:57,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:57,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36312 deadline: 1732692357116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:57,125 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:57,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:57,125 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:57,126 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:57,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36338 deadline: 1732692357116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:57,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:57,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:57,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:57,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741960_1136 (size=48139) 2024-11-27T07:24:57,128 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=92, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/7c61ad07bbbe4f3683ef9d7658394644 2024-11-27T07:24:57,141 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/79e6b404310b4e14ac394562c8b3b2e3 is 50, key is test_row_0/B:col10/1732692296395/Put/seqid=0 2024-11-27T07:24:57,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741961_1137 (size=12001) 2024-11-27T07:24:57,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:57,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36352 deadline: 1732692357230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:57,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:57,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:57,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:57,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:57,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36298 deadline: 1732692357230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:57,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:57,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36312 deadline: 1732692357230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:57,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:57,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36314 deadline: 1732692357231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:57,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:57,235 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:57,235 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:57,235 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:57,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36338 deadline: 1732692357231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:57,235 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:57,235 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:57,235 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:57,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:57,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:57,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:57,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:57,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:57,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:57,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-27T07:24:57,339 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T07:24:57,340 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T07:24:57,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-27T07:24:57,342 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T07:24:57,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-27T07:24:57,343 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T07:24:57,344 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T07:24:57,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:57,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36352 deadline: 1732692357439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:57,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:57,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36314 deadline: 1732692357440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:57,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:57,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36338 deadline: 1732692357440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:57,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:57,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:57,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:57,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:57,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:57,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:57,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:57,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:57,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:57,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:57,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:57,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36312 deadline: 1732692357442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:57,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36298 deadline: 1732692357442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:57,443 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:57,443 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:57,443 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:57,443 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:57,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:57,444 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:57,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-27T07:24:57,451 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/6c90e21305ac40da83d99fa62732443b as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/6c90e21305ac40da83d99fa62732443b 2024-11-27T07:24:57,460 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 225833644b1bebd5fa53ae861d7c7070/A of 225833644b1bebd5fa53ae861d7c7070 into 6c90e21305ac40da83d99fa62732443b(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:24:57,460 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 225833644b1bebd5fa53ae861d7c7070: 2024-11-27T07:24:57,460 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., storeName=225833644b1bebd5fa53ae861d7c7070/A, priority=12, startTime=1732692296941; duration=0sec 2024-11-27T07:24:57,460 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:24:57,460 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 225833644b1bebd5fa53ae861d7c7070:A 2024-11-27T07:24:57,479 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/2276c55338654b50a52106db051a31e7 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/2276c55338654b50a52106db051a31e7 2024-11-27T07:24:57,485 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 225833644b1bebd5fa53ae861d7c7070/C of 225833644b1bebd5fa53ae861d7c7070 into 2276c55338654b50a52106db051a31e7(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:24:57,485 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 225833644b1bebd5fa53ae861d7c7070: 2024-11-27T07:24:57,485 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., storeName=225833644b1bebd5fa53ae861d7c7070/C, priority=12, startTime=1732692296943; duration=0sec 2024-11-27T07:24:57,485 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:24:57,485 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 225833644b1bebd5fa53ae861d7c7070:C 2024-11-27T07:24:57,498 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-27T07:24:57,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:24:57,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. as already flushing 2024-11-27T07:24:57,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:24:57,498 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:57,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:57,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:57,558 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/79e6b404310b4e14ac394562c8b3b2e3 2024-11-27T07:24:57,570 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/8754fa27f2fe4423a45531f7849a375e is 50, key is test_row_0/C:col10/1732692296395/Put/seqid=0 2024-11-27T07:24:57,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741962_1138 (size=12001) 2024-11-27T07:24:57,600 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/8754fa27f2fe4423a45531f7849a375e 2024-11-27T07:24:57,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/7c61ad07bbbe4f3683ef9d7658394644 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/7c61ad07bbbe4f3683ef9d7658394644 2024-11-27T07:24:57,621 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/7c61ad07bbbe4f3683ef9d7658394644, entries=250, sequenceid=92, filesize=47.0 K 2024-11-27T07:24:57,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/79e6b404310b4e14ac394562c8b3b2e3 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/79e6b404310b4e14ac394562c8b3b2e3 2024-11-27T07:24:57,629 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/79e6b404310b4e14ac394562c8b3b2e3, entries=150, sequenceid=92, filesize=11.7 K 2024-11-27T07:24:57,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/8754fa27f2fe4423a45531f7849a375e as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/8754fa27f2fe4423a45531f7849a375e 2024-11-27T07:24:57,636 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/8754fa27f2fe4423a45531f7849a375e, entries=150, sequenceid=92, filesize=11.7 K 2024-11-27T07:24:57,637 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 225833644b1bebd5fa53ae861d7c7070 in 593ms, sequenceid=92, compaction requested=false 2024-11-27T07:24:57,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 225833644b1bebd5fa53ae861d7c7070: 2024-11-27T07:24:57,652 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-27T07:24:57,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:24:57,652 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 225833644b1bebd5fa53ae861d7c7070 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-27T07:24:57,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-27T07:24:57,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411275ea753432bfe4e128832e53872ceaea0_225833644b1bebd5fa53ae861d7c7070 is 50, key is test_row_0/A:col10/1732692297103/Put/seqid=0 2024-11-27T07:24:57,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741963_1139 (size=12154) 2024-11-27T07:24:57,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:24:57,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. as already flushing 2024-11-27T07:24:57,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:57,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36314 deadline: 1732692357758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:57,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:57,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:57,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:57,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:57,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36312 deadline: 1732692357759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:57,767 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:57,767 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:57,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:57,767 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:57,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36298 deadline: 1732692357759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:57,768 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:57,768 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:57,768 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:57,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:57,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36352 deadline: 1732692357764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:57,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:57,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:57,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:57,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:57,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36338 deadline: 1732692357766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:57,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:57,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:57,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:57,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:57,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36314 deadline: 1732692357869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:57,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:57,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:57,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36312 deadline: 1732692357870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:57,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:57,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:57,873 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:57,873 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:57,873 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:57,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:57,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36298 deadline: 1732692357879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:57,880 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:57,880 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:57,880 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:57,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:57,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36352 deadline: 1732692357879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:57,880 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:57,880 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:57,880 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:57,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:57,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36338 deadline: 1732692357899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:57,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:57,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:57,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:57,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-27T07:24:58,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:58,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:58,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36314 deadline: 1732692358080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:58,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:58,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36312 deadline: 1732692358080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:58,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:58,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:58,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:58,081 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:58,081 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:58,081 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:58,084 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411275ea753432bfe4e128832e53872ceaea0_225833644b1bebd5fa53ae861d7c7070 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411275ea753432bfe4e128832e53872ceaea0_225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:24:58,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/940f4c44b1c044dfbeb89c0c6988c8e8, store: [table=TestAcidGuarantees family=A region=225833644b1bebd5fa53ae861d7c7070] 2024-11-27T07:24:58,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/940f4c44b1c044dfbeb89c0c6988c8e8 is 175, key is test_row_0/A:col10/1732692297103/Put/seqid=0 2024-11-27T07:24:58,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:58,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:58,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36352 deadline: 1732692358089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:58,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36298 deadline: 1732692358089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:58,091 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:58,091 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:58,091 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:58,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:58,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:58,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:58,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741964_1140 (size=30955) 2024-11-27T07:24:58,097 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=120, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/940f4c44b1c044dfbeb89c0c6988c8e8 2024-11-27T07:24:58,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:58,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36338 deadline: 1732692358110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:58,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:58,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:58,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/5ce9338c195541b6aa715529e4696e08 is 50, key is test_row_0/B:col10/1732692297103/Put/seqid=0 2024-11-27T07:24:58,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:58,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741965_1141 (size=12001) 2024-11-27T07:24:58,137 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/5ce9338c195541b6aa715529e4696e08 2024-11-27T07:24:58,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/c3eec9f4933a4b29a4874ff2c96443a3 is 50, key is test_row_0/C:col10/1732692297103/Put/seqid=0 2024-11-27T07:24:58,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741966_1142 (size=12001) 2024-11-27T07:24:58,195 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/c3eec9f4933a4b29a4874ff2c96443a3 2024-11-27T07:24:58,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/940f4c44b1c044dfbeb89c0c6988c8e8 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/940f4c44b1c044dfbeb89c0c6988c8e8 2024-11-27T07:24:58,211 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/940f4c44b1c044dfbeb89c0c6988c8e8, entries=150, sequenceid=120, filesize=30.2 K 2024-11-27T07:24:58,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/5ce9338c195541b6aa715529e4696e08 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/5ce9338c195541b6aa715529e4696e08 2024-11-27T07:24:58,220 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/5ce9338c195541b6aa715529e4696e08, entries=150, sequenceid=120, filesize=11.7 K 2024-11-27T07:24:58,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/c3eec9f4933a4b29a4874ff2c96443a3 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/c3eec9f4933a4b29a4874ff2c96443a3 2024-11-27T07:24:58,229 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/c3eec9f4933a4b29a4874ff2c96443a3, entries=150, sequenceid=120, filesize=11.7 K 2024-11-27T07:24:58,230 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 225833644b1bebd5fa53ae861d7c7070 in 578ms, sequenceid=120, compaction requested=true 2024-11-27T07:24:58,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 225833644b1bebd5fa53ae861d7c7070: 2024-11-27T07:24:58,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:24:58,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-27T07:24:58,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-27T07:24:58,235 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-27T07:24:58,235 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 889 msec 2024-11-27T07:24:58,237 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 895 msec 2024-11-27T07:24:58,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:24:58,392 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T07:24:58,392 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 225833644b1bebd5fa53ae861d7c7070 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-27T07:24:58,406 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127098043b48ce440fd8aa8c458345a4fae_225833644b1bebd5fa53ae861d7c7070 is 50, key is test_row_0/A:col10/1732692297749/Put/seqid=0 2024-11-27T07:24:58,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741967_1143 (size=14694) 2024-11-27T07:24:58,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:58,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36314 deadline: 1732692358459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:58,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:58,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36352 deadline: 1732692358462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:58,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:58,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:58,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:58,470 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:58,470 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:58,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:58,470 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:58,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36338 deadline: 1732692358465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:58,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:58,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:58,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:58,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:58,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36298 deadline: 1732692358467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:58,472 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:58,472 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:58,472 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:58,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:58,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36312 deadline: 1732692358467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:58,473 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:58,473 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:58,473 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:58,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-27T07:24:58,480 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T07:24:58,480 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T07:24:58,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-27T07:24:58,482 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T07:24:58,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-27T07:24:58,483 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T07:24:58,483 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T07:24:58,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:58,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36352 deadline: 1732692358579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:58,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:58,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36298 deadline: 1732692358579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:58,584 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:58,584 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:58,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:58,585 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:58,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36314 deadline: 1732692358579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:58,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:58,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:58,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:58,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:58,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:58,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:58,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:58,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36338 deadline: 1732692358579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:58,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:58,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36312 deadline: 1732692358580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:58,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:58,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:58,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:58,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:58,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:58,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:58,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-27T07:24:58,637 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-27T07:24:58,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:24:58,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. as already flushing 2024-11-27T07:24:58,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:24:58,637 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:58,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:58,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:58,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:58,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:58,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36298 deadline: 1732692358789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:58,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36352 deadline: 1732692358789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:58,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:58,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:58,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:58,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:58,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:58,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36314 deadline: 1732692358789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:58,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36312 deadline: 1732692358789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:58,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:58,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36338 deadline: 1732692358789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:58,794 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:58,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:58,794 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:58,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:58,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:58,794 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:58,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:58,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:58,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:58,795 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:58,795 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:58,795 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:58,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-27T07:24:58,799 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-27T07:24:58,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:24:58,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. as already flushing 2024-11-27T07:24:58,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:24:58,799 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:58,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:58,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:58,847 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:24:58,852 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127098043b48ce440fd8aa8c458345a4fae_225833644b1bebd5fa53ae861d7c7070 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127098043b48ce440fd8aa8c458345a4fae_225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:24:58,855 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/1b941f695de147639c5ea5ec69814fb3, store: [table=TestAcidGuarantees family=A region=225833644b1bebd5fa53ae861d7c7070] 2024-11-27T07:24:58,855 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/1b941f695de147639c5ea5ec69814fb3 is 175, key is test_row_0/A:col10/1732692297749/Put/seqid=0 2024-11-27T07:24:58,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741968_1144 (size=39649) 2024-11-27T07:24:58,955 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-27T07:24:58,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:24:58,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. as already flushing 2024-11-27T07:24:58,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:24:58,956 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:58,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:58,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:59,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:59,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36312 deadline: 1732692359099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:59,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:59,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:59,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:59,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:59,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36352 deadline: 1732692359099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:59,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:59,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:59,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:59,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:59,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36314 deadline: 1732692359099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:59,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:59,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:59,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:59,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36338 deadline: 1732692359100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:59,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:59,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:59,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:59,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:59,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:59,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36298 deadline: 1732692359101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:59,104 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:59,104 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:59,104 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:59,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-27T07:24:59,110 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-27T07:24:59,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:24:59,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. as already flushing 2024-11-27T07:24:59,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:24:59,110 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:59,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:59,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:59,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-27T07:24:59,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:24:59,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. as already flushing 2024-11-27T07:24:59,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:24:59,270 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:59,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:59,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:59,274 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=131, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/1b941f695de147639c5ea5ec69814fb3 2024-11-27T07:24:59,289 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/4e6df53335d34f58ae16edddf9061526 is 50, key is test_row_0/B:col10/1732692297749/Put/seqid=0 2024-11-27T07:24:59,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741969_1145 (size=12101) 2024-11-27T07:24:59,312 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/4e6df53335d34f58ae16edddf9061526 2024-11-27T07:24:59,338 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/51fb5a4288d04cfcb61cf14210d7b1ca is 50, key is test_row_0/C:col10/1732692297749/Put/seqid=0 2024-11-27T07:24:59,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741970_1146 (size=12101) 2024-11-27T07:24:59,361 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/51fb5a4288d04cfcb61cf14210d7b1ca 2024-11-27T07:24:59,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/1b941f695de147639c5ea5ec69814fb3 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/1b941f695de147639c5ea5ec69814fb3 2024-11-27T07:24:59,381 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/1b941f695de147639c5ea5ec69814fb3, entries=200, sequenceid=131, filesize=38.7 K 2024-11-27T07:24:59,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/4e6df53335d34f58ae16edddf9061526 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/4e6df53335d34f58ae16edddf9061526 2024-11-27T07:24:59,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-27T07:24:59,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:24:59,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. as already flushing 2024-11-27T07:24:59,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:24:59,432 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:59,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:59,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:24:59,457 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/4e6df53335d34f58ae16edddf9061526, entries=150, sequenceid=131, filesize=11.8 K 2024-11-27T07:24:59,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/51fb5a4288d04cfcb61cf14210d7b1ca as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/51fb5a4288d04cfcb61cf14210d7b1ca 2024-11-27T07:24:59,492 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/51fb5a4288d04cfcb61cf14210d7b1ca, entries=150, sequenceid=131, filesize=11.8 K 2024-11-27T07:24:59,494 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 225833644b1bebd5fa53ae861d7c7070 in 1102ms, sequenceid=131, compaction requested=true 2024-11-27T07:24:59,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 225833644b1bebd5fa53ae861d7c7070: 2024-11-27T07:24:59,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 225833644b1bebd5fa53ae861d7c7070:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T07:24:59,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:24:59,494 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T07:24:59,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 225833644b1bebd5fa53ae861d7c7070:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T07:24:59,494 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T07:24:59,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:24:59,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 225833644b1bebd5fa53ae861d7c7070:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T07:24:59,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:24:59,497 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 149836 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T07:24:59,497 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48242 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T07:24:59,497 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1541): 225833644b1bebd5fa53ae861d7c7070/A is initiating minor compaction (all files) 2024-11-27T07:24:59,497 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 225833644b1bebd5fa53ae861d7c7070/B is initiating minor compaction (all files) 2024-11-27T07:24:59,497 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 225833644b1bebd5fa53ae861d7c7070/B in TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:24:59,497 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 225833644b1bebd5fa53ae861d7c7070/A in TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:24:59,497 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/f5fe59552cb748e983308e4283c0e01f, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/79e6b404310b4e14ac394562c8b3b2e3, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/5ce9338c195541b6aa715529e4696e08, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/4e6df53335d34f58ae16edddf9061526] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp, totalSize=47.1 K 2024-11-27T07:24:59,497 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/6c90e21305ac40da83d99fa62732443b, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/7c61ad07bbbe4f3683ef9d7658394644, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/940f4c44b1c044dfbeb89c0c6988c8e8, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/1b941f695de147639c5ea5ec69814fb3] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp, totalSize=146.3 K 2024-11-27T07:24:59,497 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:24:59,497 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. files: [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/6c90e21305ac40da83d99fa62732443b, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/7c61ad07bbbe4f3683ef9d7658394644, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/940f4c44b1c044dfbeb89c0c6988c8e8, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/1b941f695de147639c5ea5ec69814fb3] 2024-11-27T07:24:59,499 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c90e21305ac40da83d99fa62732443b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732692296382 2024-11-27T07:24:59,499 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting f5fe59552cb748e983308e4283c0e01f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732692296382 2024-11-27T07:24:59,499 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 79e6b404310b4e14ac394562c8b3b2e3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732692296395 2024-11-27T07:24:59,499 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c61ad07bbbe4f3683ef9d7658394644, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732692296395 2024-11-27T07:24:59,500 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ce9338c195541b6aa715529e4696e08, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732692297103 2024-11-27T07:24:59,500 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 940f4c44b1c044dfbeb89c0c6988c8e8, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732692297103 2024-11-27T07:24:59,503 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b941f695de147639c5ea5ec69814fb3, keycount=200, bloomtype=ROW, size=38.7 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732692297749 2024-11-27T07:24:59,504 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e6df53335d34f58ae16edddf9061526, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732692297749 2024-11-27T07:24:59,523 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=225833644b1bebd5fa53ae861d7c7070] 2024-11-27T07:24:59,541 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 225833644b1bebd5fa53ae861d7c7070#B#compaction#124 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:24:59,541 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/fb0eb8e459c74b7a9944ad9f1098deea is 50, key is test_row_0/B:col10/1732692297749/Put/seqid=0 2024-11-27T07:24:59,543 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411271fb3a0fdd1394cc39b472922d04169e1_225833644b1bebd5fa53ae861d7c7070 store=[table=TestAcidGuarantees family=A region=225833644b1bebd5fa53ae861d7c7070] 2024-11-27T07:24:59,547 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411271fb3a0fdd1394cc39b472922d04169e1_225833644b1bebd5fa53ae861d7c7070, store=[table=TestAcidGuarantees family=A region=225833644b1bebd5fa53ae861d7c7070] 2024-11-27T07:24:59,547 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411271fb3a0fdd1394cc39b472922d04169e1_225833644b1bebd5fa53ae861d7c7070 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=225833644b1bebd5fa53ae861d7c7070] 2024-11-27T07:24:59,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741971_1147 (size=12375) 2024-11-27T07:24:59,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741972_1148 (size=4469) 2024-11-27T07:24:59,586 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-27T07:24:59,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:24:59,586 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 225833644b1bebd5fa53ae861d7c7070 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-27T07:24:59,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127bfe07aa57b754b25939a5944d488e2c1_225833644b1bebd5fa53ae861d7c7070 is 50, key is test_row_0/A:col10/1732692298464/Put/seqid=0 2024-11-27T07:24:59,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:24:59,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. as already flushing 2024-11-27T07:24:59,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-27T07:24:59,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741973_1149 (size=12304) 2024-11-27T07:24:59,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:59,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36314 deadline: 1732692359618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:59,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:59,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:59,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36298 deadline: 1732692359619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:59,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:59,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:59,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:59,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36338 deadline: 1732692359619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:59,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:59,625 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:59,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:59,626 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:59,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:59,626 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:59,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:59,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36312 deadline: 1732692359624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:59,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:59,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:59,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:59,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:59,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36352 deadline: 1732692359624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:59,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:59,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:59,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:59,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:59,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36314 deadline: 1732692359731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:59,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:59,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:59,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:59,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:59,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36298 deadline: 1732692359732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:59,736 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:59,736 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:59,736 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:59,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:59,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36312 deadline: 1732692359733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:59,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:59,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:59,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:59,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:59,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36352 deadline: 1732692359739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:59,742 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:59,742 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:59,742 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:59,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:59,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36298 deadline: 1732692359939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:59,944 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:59,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:59,944 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:59,944 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:59,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36314 deadline: 1732692359939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:59,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:59,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:59,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:59,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36312 deadline: 1732692359940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:59,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:59,945 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:59,945 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:59,945 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:59,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:24:59,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36352 deadline: 1732692359949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:24:59,950 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:59,950 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:24:59,950 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:24:59,977 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/fb0eb8e459c74b7a9944ad9f1098deea as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/fb0eb8e459c74b7a9944ad9f1098deea 2024-11-27T07:24:59,983 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 225833644b1bebd5fa53ae861d7c7070#A#compaction#123 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:24:59,984 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/9d8168d9b6ab4d5e93fee0bbaaee8dd3 is 175, key is test_row_0/A:col10/1732692297749/Put/seqid=0 2024-11-27T07:24:59,988 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 225833644b1bebd5fa53ae861d7c7070/B of 225833644b1bebd5fa53ae861d7c7070 into fb0eb8e459c74b7a9944ad9f1098deea(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:24:59,988 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 225833644b1bebd5fa53ae861d7c7070: 2024-11-27T07:24:59,988 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., storeName=225833644b1bebd5fa53ae861d7c7070/B, priority=12, startTime=1732692299494; duration=0sec 2024-11-27T07:24:59,988 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:24:59,988 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 225833644b1bebd5fa53ae861d7c7070:B 2024-11-27T07:24:59,988 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T07:24:59,990 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48242 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T07:24:59,990 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 225833644b1bebd5fa53ae861d7c7070/C is initiating minor compaction (all files) 2024-11-27T07:24:59,990 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 225833644b1bebd5fa53ae861d7c7070/C in TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:24:59,990 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/2276c55338654b50a52106db051a31e7, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/8754fa27f2fe4423a45531f7849a375e, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/c3eec9f4933a4b29a4874ff2c96443a3, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/51fb5a4288d04cfcb61cf14210d7b1ca] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp, totalSize=47.1 K 2024-11-27T07:24:59,991 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 2276c55338654b50a52106db051a31e7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732692296382 2024-11-27T07:24:59,991 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 8754fa27f2fe4423a45531f7849a375e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732692296395 2024-11-27T07:24:59,992 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting c3eec9f4933a4b29a4874ff2c96443a3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732692297103 2024-11-27T07:24:59,994 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 51fb5a4288d04cfcb61cf14210d7b1ca, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732692297749 2024-11-27T07:24:59,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741974_1150 (size=31329) 2024-11-27T07:25:00,007 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/9d8168d9b6ab4d5e93fee0bbaaee8dd3 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/9d8168d9b6ab4d5e93fee0bbaaee8dd3 2024-11-27T07:25:00,009 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 225833644b1bebd5fa53ae861d7c7070#C#compaction#126 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:25:00,009 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/ef60204e57024d428fb4e501961dad51 is 50, key is test_row_0/C:col10/1732692297749/Put/seqid=0 2024-11-27T07:25:00,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741975_1151 (size=12375) 2024-11-27T07:25:00,018 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 225833644b1bebd5fa53ae861d7c7070/A of 225833644b1bebd5fa53ae861d7c7070 into 9d8168d9b6ab4d5e93fee0bbaaee8dd3(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:25:00,018 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 225833644b1bebd5fa53ae861d7c7070: 2024-11-27T07:25:00,018 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., storeName=225833644b1bebd5fa53ae861d7c7070/A, priority=12, startTime=1732692299494; duration=0sec 2024-11-27T07:25:00,018 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:25:00,018 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 225833644b1bebd5fa53ae861d7c7070:A 2024-11-27T07:25:00,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:25:00,026 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/ef60204e57024d428fb4e501961dad51 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/ef60204e57024d428fb4e501961dad51 2024-11-27T07:25:00,031 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127bfe07aa57b754b25939a5944d488e2c1_225833644b1bebd5fa53ae861d7c7070 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127bfe07aa57b754b25939a5944d488e2c1_225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:25:00,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/f3ad16b9b08c42baa75299574525b972, store: [table=TestAcidGuarantees family=A region=225833644b1bebd5fa53ae861d7c7070] 2024-11-27T07:25:00,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/f3ad16b9b08c42baa75299574525b972 is 175, key is test_row_0/A:col10/1732692298464/Put/seqid=0 2024-11-27T07:25:00,036 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 225833644b1bebd5fa53ae861d7c7070/C of 225833644b1bebd5fa53ae861d7c7070 into ef60204e57024d428fb4e501961dad51(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:25:00,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741976_1152 (size=31105) 2024-11-27T07:25:00,037 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 225833644b1bebd5fa53ae861d7c7070: 2024-11-27T07:25:00,037 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., storeName=225833644b1bebd5fa53ae861d7c7070/C, priority=12, startTime=1732692299494; duration=0sec 2024-11-27T07:25:00,037 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:25:00,037 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 225833644b1bebd5fa53ae861d7c7070:C 2024-11-27T07:25:00,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:00,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36298 deadline: 1732692360249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:00,252 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:00,252 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:00,252 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:00,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:00,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36312 deadline: 1732692360249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:00,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:00,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36314 deadline: 1732692360249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:00,254 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:00,254 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:00,254 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:00,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:00,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:00,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:00,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:00,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36352 deadline: 1732692360259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:00,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:00,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:00,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:00,437 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=156, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/f3ad16b9b08c42baa75299574525b972 2024-11-27T07:25:00,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/3468c3fd380f461ba5417f79aa1393b7 is 50, key is test_row_0/B:col10/1732692298464/Put/seqid=0 2024-11-27T07:25:00,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741977_1153 (size=12151) 2024-11-27T07:25:00,483 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/3468c3fd380f461ba5417f79aa1393b7 2024-11-27T07:25:00,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/a7b7f21cb1e846ebb8171444ce12c453 is 50, key is test_row_0/C:col10/1732692298464/Put/seqid=0 2024-11-27T07:25:00,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741978_1154 (size=12151) 2024-11-27T07:25:00,500 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/a7b7f21cb1e846ebb8171444ce12c453 2024-11-27T07:25:00,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/f3ad16b9b08c42baa75299574525b972 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/f3ad16b9b08c42baa75299574525b972 2024-11-27T07:25:00,521 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/f3ad16b9b08c42baa75299574525b972, entries=150, sequenceid=156, filesize=30.4 K 2024-11-27T07:25:00,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/3468c3fd380f461ba5417f79aa1393b7 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/3468c3fd380f461ba5417f79aa1393b7 2024-11-27T07:25:00,537 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/3468c3fd380f461ba5417f79aa1393b7, entries=150, sequenceid=156, filesize=11.9 K 2024-11-27T07:25:00,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/a7b7f21cb1e846ebb8171444ce12c453 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/a7b7f21cb1e846ebb8171444ce12c453 2024-11-27T07:25:00,552 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/a7b7f21cb1e846ebb8171444ce12c453, entries=150, sequenceid=156, filesize=11.9 K 2024-11-27T07:25:00,553 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 225833644b1bebd5fa53ae861d7c7070 in 967ms, sequenceid=156, compaction requested=false 2024-11-27T07:25:00,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 225833644b1bebd5fa53ae861d7c7070: 2024-11-27T07:25:00,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:00,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-27T07:25:00,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-27T07:25:00,557 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-27T07:25:00,557 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0720 sec 2024-11-27T07:25:00,559 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 2.0780 sec 2024-11-27T07:25:00,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-27T07:25:00,630 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T07:25:00,630 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T07:25:00,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-27T07:25:00,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-27T07:25:00,634 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T07:25:00,635 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T07:25:00,635 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T07:25:00,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:25:00,640 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T07:25:00,640 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 225833644b1bebd5fa53ae861d7c7070 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-27T07:25:00,649 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127496b4dc66f83464095c1de12922bcbfc_225833644b1bebd5fa53ae861d7c7070 is 50, key is test_row_0/A:col10/1732692299616/Put/seqid=0 2024-11-27T07:25:00,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741979_1155 (size=14794) 2024-11-27T07:25:00,658 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:25:00,673 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127496b4dc66f83464095c1de12922bcbfc_225833644b1bebd5fa53ae861d7c7070 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127496b4dc66f83464095c1de12922bcbfc_225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:25:00,676 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/6e2ab6dd0050461e9b329d880f47f866, store: [table=TestAcidGuarantees family=A region=225833644b1bebd5fa53ae861d7c7070] 2024-11-27T07:25:00,677 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/6e2ab6dd0050461e9b329d880f47f866 is 175, key is test_row_0/A:col10/1732692299616/Put/seqid=0 2024-11-27T07:25:00,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741980_1156 (size=39749) 2024-11-27T07:25:00,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:00,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36338 deadline: 1732692360729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:00,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:00,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:00,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:00,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-27T07:25:00,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:00,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36298 deadline: 1732692360759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:00,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:00,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36314 deadline: 1732692360759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:00,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:00,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36312 deadline: 1732692360759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:00,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:00,765 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:00,765 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:00,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:00,765 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:00,766 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:00,766 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:00,766 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:00,766 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:00,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:00,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36352 deadline: 1732692360769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:00,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:00,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:00,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:00,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-27T07:25:00,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:00,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. as already flushing 2024-11-27T07:25:00,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:00,790 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:00,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:00,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:00,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:00,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36338 deadline: 1732692360839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:00,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:00,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:00,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:00,942 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-27T07:25:00,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:00,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. as already flushing 2024-11-27T07:25:00,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:00,943 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:00,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:00,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:00,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-27T07:25:01,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:01,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36338 deadline: 1732692361048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:01,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:01,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:01,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:01,093 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=172, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/6e2ab6dd0050461e9b329d880f47f866 2024-11-27T07:25:01,095 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-27T07:25:01,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:01,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. as already flushing 2024-11-27T07:25:01,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:01,096 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:01,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:01,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:01,103 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/613cfd71f60741bdb8f44c3e28e6138d is 50, key is test_row_0/B:col10/1732692299616/Put/seqid=0 2024-11-27T07:25:01,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741981_1157 (size=12151) 2024-11-27T07:25:01,121 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/613cfd71f60741bdb8f44c3e28e6138d 2024-11-27T07:25:01,139 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/8b63ee95925f4d7d8538ede3124c0363 is 50, key is test_row_0/C:col10/1732692299616/Put/seqid=0 2024-11-27T07:25:01,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741982_1158 (size=12151) 2024-11-27T07:25:01,151 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/8b63ee95925f4d7d8538ede3124c0363 2024-11-27T07:25:01,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/6e2ab6dd0050461e9b329d880f47f866 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/6e2ab6dd0050461e9b329d880f47f866 2024-11-27T07:25:01,164 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/6e2ab6dd0050461e9b329d880f47f866, entries=200, sequenceid=172, filesize=38.8 K 2024-11-27T07:25:01,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/613cfd71f60741bdb8f44c3e28e6138d as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/613cfd71f60741bdb8f44c3e28e6138d 2024-11-27T07:25:01,173 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/613cfd71f60741bdb8f44c3e28e6138d, entries=150, sequenceid=172, filesize=11.9 K 2024-11-27T07:25:01,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/8b63ee95925f4d7d8538ede3124c0363 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/8b63ee95925f4d7d8538ede3124c0363 2024-11-27T07:25:01,187 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/8b63ee95925f4d7d8538ede3124c0363, entries=150, sequenceid=172, filesize=11.9 K 2024-11-27T07:25:01,188 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 225833644b1bebd5fa53ae861d7c7070 in 548ms, sequenceid=172, compaction requested=true 2024-11-27T07:25:01,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 225833644b1bebd5fa53ae861d7c7070: 2024-11-27T07:25:01,189 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:25:01,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 225833644b1bebd5fa53ae861d7c7070:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T07:25:01,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:25:01,190 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:25:01,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 225833644b1bebd5fa53ae861d7c7070:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T07:25:01,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:25:01,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 225833644b1bebd5fa53ae861d7c7070:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T07:25:01,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:25:01,191 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36677 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:25:01,191 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102183 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:25:01,191 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 225833644b1bebd5fa53ae861d7c7070/B is initiating minor compaction (all files) 2024-11-27T07:25:01,191 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1541): 225833644b1bebd5fa53ae861d7c7070/A is initiating minor compaction (all files) 2024-11-27T07:25:01,191 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 225833644b1bebd5fa53ae861d7c7070/B in TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:01,191 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/fb0eb8e459c74b7a9944ad9f1098deea, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/3468c3fd380f461ba5417f79aa1393b7, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/613cfd71f60741bdb8f44c3e28e6138d] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp, totalSize=35.8 K 2024-11-27T07:25:01,192 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 225833644b1bebd5fa53ae861d7c7070/A in TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:01,192 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/9d8168d9b6ab4d5e93fee0bbaaee8dd3, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/f3ad16b9b08c42baa75299574525b972, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/6e2ab6dd0050461e9b329d880f47f866] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp, totalSize=99.8 K 2024-11-27T07:25:01,192 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:01,192 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. files: [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/9d8168d9b6ab4d5e93fee0bbaaee8dd3, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/f3ad16b9b08c42baa75299574525b972, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/6e2ab6dd0050461e9b329d880f47f866] 2024-11-27T07:25:01,192 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting fb0eb8e459c74b7a9944ad9f1098deea, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732692297749 2024-11-27T07:25:01,193 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d8168d9b6ab4d5e93fee0bbaaee8dd3, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732692297749 2024-11-27T07:25:01,194 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3ad16b9b08c42baa75299574525b972, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732692298455 2024-11-27T07:25:01,194 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 3468c3fd380f461ba5417f79aa1393b7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732692298455 2024-11-27T07:25:01,194 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e2ab6dd0050461e9b329d880f47f866, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732692299616 2024-11-27T07:25:01,195 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 613cfd71f60741bdb8f44c3e28e6138d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732692299616 2024-11-27T07:25:01,210 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 225833644b1bebd5fa53ae861d7c7070#B#compaction#132 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:25:01,211 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/c57c343b78954d3e9e64e79f44800226 is 50, key is test_row_0/B:col10/1732692299616/Put/seqid=0 2024-11-27T07:25:01,246 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=225833644b1bebd5fa53ae861d7c7070] 2024-11-27T07:25:01,250 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-27T07:25:01,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:01,251 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 225833644b1bebd5fa53ae861d7c7070 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-27T07:25:01,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-27T07:25:01,265 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112751cb61c0c5af48f2b8e753b4b6536197_225833644b1bebd5fa53ae861d7c7070 store=[table=TestAcidGuarantees family=A region=225833644b1bebd5fa53ae861d7c7070] 2024-11-27T07:25:01,267 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112751cb61c0c5af48f2b8e753b4b6536197_225833644b1bebd5fa53ae861d7c7070, store=[table=TestAcidGuarantees family=A region=225833644b1bebd5fa53ae861d7c7070] 2024-11-27T07:25:01,267 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112751cb61c0c5af48f2b8e753b4b6536197_225833644b1bebd5fa53ae861d7c7070 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=225833644b1bebd5fa53ae861d7c7070] 2024-11-27T07:25:01,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741983_1159 (size=12527) 2024-11-27T07:25:01,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411278ef2f316c910481cb5d5b657feaf364e_225833644b1bebd5fa53ae861d7c7070 is 50, key is test_row_0/A:col10/1732692300727/Put/seqid=0 2024-11-27T07:25:01,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741984_1160 (size=4469) 2024-11-27T07:25:01,334 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 225833644b1bebd5fa53ae861d7c7070#A#compaction#133 average throughput is 0.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:25:01,334 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/9703987f37e044c3899aaad633b1da47 is 175, key is test_row_0/A:col10/1732692299616/Put/seqid=0 2024-11-27T07:25:01,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741985_1161 (size=12304) 2024-11-27T07:25:01,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:25:01,347 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411278ef2f316c910481cb5d5b657feaf364e_225833644b1bebd5fa53ae861d7c7070 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411278ef2f316c910481cb5d5b657feaf364e_225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:25:01,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/88b98d16c4c242bcb6433521083bd2af, store: [table=TestAcidGuarantees family=A region=225833644b1bebd5fa53ae861d7c7070] 2024-11-27T07:25:01,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/88b98d16c4c242bcb6433521083bd2af is 175, key is test_row_0/A:col10/1732692300727/Put/seqid=0 2024-11-27T07:25:01,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. as already flushing 2024-11-27T07:25:01,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:25:01,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741986_1162 (size=31481) 2024-11-27T07:25:01,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741987_1163 (size=31105) 2024-11-27T07:25:01,405 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/9703987f37e044c3899aaad633b1da47 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/9703987f37e044c3899aaad633b1da47 2024-11-27T07:25:01,411 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 225833644b1bebd5fa53ae861d7c7070/A of 225833644b1bebd5fa53ae861d7c7070 into 9703987f37e044c3899aaad633b1da47(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:25:01,412 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 225833644b1bebd5fa53ae861d7c7070: 2024-11-27T07:25:01,412 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., storeName=225833644b1bebd5fa53ae861d7c7070/A, priority=13, startTime=1732692301189; duration=0sec 2024-11-27T07:25:01,412 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:25:01,412 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 225833644b1bebd5fa53ae861d7c7070:A 2024-11-27T07:25:01,412 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:25:01,415 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36677 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:25:01,415 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1541): 225833644b1bebd5fa53ae861d7c7070/C is initiating minor compaction (all files) 2024-11-27T07:25:01,415 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 225833644b1bebd5fa53ae861d7c7070/C in TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:01,415 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/ef60204e57024d428fb4e501961dad51, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/a7b7f21cb1e846ebb8171444ce12c453, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/8b63ee95925f4d7d8538ede3124c0363] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp, totalSize=35.8 K 2024-11-27T07:25:01,416 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef60204e57024d428fb4e501961dad51, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732692297749 2024-11-27T07:25:01,417 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7b7f21cb1e846ebb8171444ce12c453, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732692298455 2024-11-27T07:25:01,417 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b63ee95925f4d7d8538ede3124c0363, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732692299616 2024-11-27T07:25:01,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:01,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36338 deadline: 1732692361418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:01,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:01,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:01,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:01,430 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 225833644b1bebd5fa53ae861d7c7070#C#compaction#135 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:25:01,431 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/42cce269fb91498abaedfffd0a9b0fc2 is 50, key is test_row_0/C:col10/1732692299616/Put/seqid=0 2024-11-27T07:25:01,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741988_1164 (size=12527) 2024-11-27T07:25:01,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:01,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36338 deadline: 1732692361529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:01,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:01,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:01,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:01,678 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/c57c343b78954d3e9e64e79f44800226 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/c57c343b78954d3e9e64e79f44800226 2024-11-27T07:25:01,683 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 225833644b1bebd5fa53ae861d7c7070/B of 225833644b1bebd5fa53ae861d7c7070 into c57c343b78954d3e9e64e79f44800226(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:25:01,684 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 225833644b1bebd5fa53ae861d7c7070: 2024-11-27T07:25:01,684 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., storeName=225833644b1bebd5fa53ae861d7c7070/B, priority=13, startTime=1732692301190; duration=0sec 2024-11-27T07:25:01,684 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:25:01,684 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 225833644b1bebd5fa53ae861d7c7070:B 2024-11-27T07:25:01,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:01,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36338 deadline: 1732692361739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:01,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:01,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:01,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:01,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-27T07:25:01,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:01,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36314 deadline: 1732692361779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:01,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:01,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36298 deadline: 1732692361779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:01,780 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 289954c9202d,42117,1732692259671 for 'test_row_0' in 225833644b1bebd5fa53ae861d7c7070 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T07:25:01,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:01,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:01,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:01,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36312 deadline: 1732692361779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:01,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:01,781 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 289954c9202d,42117,1732692259671 for 'test_row_1' in 225833644b1bebd5fa53ae861d7c7070 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T07:25:01,781 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:01,781 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:01,781 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:01,781 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 289954c9202d,42117,1732692259671 for 'test_row_2' in 225833644b1bebd5fa53ae861d7c7070 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T07:25:01,781 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:01,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:01,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:01,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:01,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36352 deadline: 1732692361788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:01,789 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 289954c9202d,42117,1732692259671 for 'test_row_1' in 225833644b1bebd5fa53ae861d7c7070 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T07:25:01,790 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:01,790 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:01,790 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:01,803 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=195, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/88b98d16c4c242bcb6433521083bd2af 2024-11-27T07:25:01,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/4543c195626c4d35b19ce170276461d1 is 50, key is test_row_0/B:col10/1732692300727/Put/seqid=0 2024-11-27T07:25:01,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741989_1165 (size=12151) 2024-11-27T07:25:01,815 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/4543c195626c4d35b19ce170276461d1 2024-11-27T07:25:01,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/01330647c4d542798dcb9938ee6beac2 is 50, key is test_row_0/C:col10/1732692300727/Put/seqid=0 2024-11-27T07:25:01,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741990_1166 (size=12151) 2024-11-27T07:25:01,872 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/42cce269fb91498abaedfffd0a9b0fc2 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/42cce269fb91498abaedfffd0a9b0fc2 2024-11-27T07:25:01,877 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 225833644b1bebd5fa53ae861d7c7070/C of 225833644b1bebd5fa53ae861d7c7070 into 42cce269fb91498abaedfffd0a9b0fc2(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:25:01,877 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 225833644b1bebd5fa53ae861d7c7070: 2024-11-27T07:25:01,877 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., storeName=225833644b1bebd5fa53ae861d7c7070/C, priority=13, startTime=1732692301190; duration=0sec 2024-11-27T07:25:01,877 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:25:01,877 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 225833644b1bebd5fa53ae861d7c7070:C 2024-11-27T07:25:02,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:02,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36338 deadline: 1732692362049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:02,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:02,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:02,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:02,229 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/01330647c4d542798dcb9938ee6beac2 2024-11-27T07:25:02,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/88b98d16c4c242bcb6433521083bd2af as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/88b98d16c4c242bcb6433521083bd2af 2024-11-27T07:25:02,239 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/88b98d16c4c242bcb6433521083bd2af, entries=150, sequenceid=195, filesize=30.4 K 2024-11-27T07:25:02,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/4543c195626c4d35b19ce170276461d1 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/4543c195626c4d35b19ce170276461d1 2024-11-27T07:25:02,244 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/4543c195626c4d35b19ce170276461d1, entries=150, sequenceid=195, filesize=11.9 K 2024-11-27T07:25:02,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/01330647c4d542798dcb9938ee6beac2 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/01330647c4d542798dcb9938ee6beac2 2024-11-27T07:25:02,250 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/01330647c4d542798dcb9938ee6beac2, entries=150, sequenceid=195, filesize=11.9 K 2024-11-27T07:25:02,251 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 225833644b1bebd5fa53ae861d7c7070 in 1000ms, sequenceid=195, compaction requested=false 2024-11-27T07:25:02,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 225833644b1bebd5fa53ae861d7c7070: 2024-11-27T07:25:02,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:02,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-27T07:25:02,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-27T07:25:02,255 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-27T07:25:02,255 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6180 sec 2024-11-27T07:25:02,257 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.6260 sec 2024-11-27T07:25:02,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:25:02,561 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T07:25:02,561 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 225833644b1bebd5fa53ae861d7c7070 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-27T07:25:02,568 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127a1cd0fa243ec486382ed0ae3e1b88ba8_225833644b1bebd5fa53ae861d7c7070 is 50, key is test_row_0/A:col10/1732692301403/Put/seqid=0 2024-11-27T07:25:02,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741991_1167 (size=12304) 2024-11-27T07:25:02,573 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:25:02,578 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127a1cd0fa243ec486382ed0ae3e1b88ba8_225833644b1bebd5fa53ae861d7c7070 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127a1cd0fa243ec486382ed0ae3e1b88ba8_225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:25:02,579 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/04ea3527339c43a0811cabee284c1a09, store: [table=TestAcidGuarantees family=A region=225833644b1bebd5fa53ae861d7c7070] 2024-11-27T07:25:02,580 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/04ea3527339c43a0811cabee284c1a09 is 175, key is test_row_0/A:col10/1732692301403/Put/seqid=0 2024-11-27T07:25:02,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741992_1168 (size=31105) 2024-11-27T07:25:02,586 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=212, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/04ea3527339c43a0811cabee284c1a09 2024-11-27T07:25:02,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/c1a936766ced4e3dae4b0457ad1261e3 is 50, key is test_row_0/B:col10/1732692301403/Put/seqid=0 2024-11-27T07:25:02,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741993_1169 (size=12151) 2024-11-27T07:25:02,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:02,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36338 deadline: 1732692362625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:02,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:02,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:02,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:02,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:02,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36338 deadline: 1732692362739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:02,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:02,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:02,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:02,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-27T07:25:02,780 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T07:25:02,780 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T07:25:02,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-27T07:25:02,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-27T07:25:02,782 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T07:25:02,782 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T07:25:02,782 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T07:25:02,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-27T07:25:02,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-27T07:25:02,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:02,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. as already flushing 2024-11-27T07:25:02,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:02,935 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:02,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:02,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:02,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:02,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36338 deadline: 1732692362949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:02,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:02,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:02,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:03,002 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/c1a936766ced4e3dae4b0457ad1261e3 2024-11-27T07:25:03,013 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/9e203d913fac497297d452a0c3efaef9 is 50, key is test_row_0/C:col10/1732692301403/Put/seqid=0 2024-11-27T07:25:03,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741994_1170 (size=12151) 2024-11-27T07:25:03,090 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-27T07:25:03,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:03,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. as already flushing 2024-11-27T07:25:03,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:03,090 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:03,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:03,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:03,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-27T07:25:03,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-27T07:25:03,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:03,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. as already flushing 2024-11-27T07:25:03,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:03,247 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:03,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:03,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:03,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:03,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36338 deadline: 1732692363259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:03,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:03,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:03,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:03,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-27T07:25:03,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:03,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. as already flushing 2024-11-27T07:25:03,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:03,401 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:03,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:03,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:03,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-27T07:25:03,418 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/9e203d913fac497297d452a0c3efaef9 2024-11-27T07:25:03,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/04ea3527339c43a0811cabee284c1a09 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/04ea3527339c43a0811cabee284c1a09 2024-11-27T07:25:03,438 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/04ea3527339c43a0811cabee284c1a09, entries=150, sequenceid=212, filesize=30.4 K 2024-11-27T07:25:03,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/c1a936766ced4e3dae4b0457ad1261e3 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/c1a936766ced4e3dae4b0457ad1261e3 2024-11-27T07:25:03,447 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/c1a936766ced4e3dae4b0457ad1261e3, entries=150, sequenceid=212, filesize=11.9 K 2024-11-27T07:25:03,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/9e203d913fac497297d452a0c3efaef9 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/9e203d913fac497297d452a0c3efaef9 2024-11-27T07:25:03,458 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/9e203d913fac497297d452a0c3efaef9, entries=150, sequenceid=212, filesize=11.9 K 2024-11-27T07:25:03,463 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 225833644b1bebd5fa53ae861d7c7070 in 901ms, sequenceid=212, compaction requested=true 2024-11-27T07:25:03,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 225833644b1bebd5fa53ae861d7c7070: 2024-11-27T07:25:03,463 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:25:03,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 225833644b1bebd5fa53ae861d7c7070:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T07:25:03,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:25:03,464 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:25:03,466 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93691 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:25:03,466 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1541): 225833644b1bebd5fa53ae861d7c7070/A is initiating minor compaction (all files) 2024-11-27T07:25:03,466 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 225833644b1bebd5fa53ae861d7c7070/A in TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:03,466 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/9703987f37e044c3899aaad633b1da47, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/88b98d16c4c242bcb6433521083bd2af, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/04ea3527339c43a0811cabee284c1a09] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp, totalSize=91.5 K 2024-11-27T07:25:03,466 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:03,466 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. files: [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/9703987f37e044c3899aaad633b1da47, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/88b98d16c4c242bcb6433521083bd2af, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/04ea3527339c43a0811cabee284c1a09] 2024-11-27T07:25:03,468 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:25:03,468 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 225833644b1bebd5fa53ae861d7c7070/B is initiating minor compaction (all files) 2024-11-27T07:25:03,468 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 225833644b1bebd5fa53ae861d7c7070/B in TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:03,468 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/c57c343b78954d3e9e64e79f44800226, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/4543c195626c4d35b19ce170276461d1, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/c1a936766ced4e3dae4b0457ad1261e3] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp, totalSize=36.0 K 2024-11-27T07:25:03,469 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9703987f37e044c3899aaad633b1da47, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732692299616 2024-11-27T07:25:03,469 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting c57c343b78954d3e9e64e79f44800226, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732692299616 2024-11-27T07:25:03,470 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88b98d16c4c242bcb6433521083bd2af, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732692300711 2024-11-27T07:25:03,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 225833644b1bebd5fa53ae861d7c7070:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T07:25:03,470 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 4543c195626c4d35b19ce170276461d1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732692300711 2024-11-27T07:25:03,470 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04ea3527339c43a0811cabee284c1a09, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732692301403 2024-11-27T07:25:03,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:25:03,471 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting c1a936766ced4e3dae4b0457ad1261e3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732692301403 2024-11-27T07:25:03,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 225833644b1bebd5fa53ae861d7c7070:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T07:25:03,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:25:03,493 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=225833644b1bebd5fa53ae861d7c7070] 2024-11-27T07:25:03,505 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 225833644b1bebd5fa53ae861d7c7070#B#compaction#142 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:25:03,506 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/b57b193b60ed416f9f021037a751afd1 is 50, key is test_row_0/B:col10/1732692301403/Put/seqid=0 2024-11-27T07:25:03,517 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127d96db359f7a94c09a303517bd48e5a41_225833644b1bebd5fa53ae861d7c7070 store=[table=TestAcidGuarantees family=A region=225833644b1bebd5fa53ae861d7c7070] 2024-11-27T07:25:03,519 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127d96db359f7a94c09a303517bd48e5a41_225833644b1bebd5fa53ae861d7c7070, store=[table=TestAcidGuarantees family=A region=225833644b1bebd5fa53ae861d7c7070] 2024-11-27T07:25:03,519 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127d96db359f7a94c09a303517bd48e5a41_225833644b1bebd5fa53ae861d7c7070 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=225833644b1bebd5fa53ae861d7c7070] 2024-11-27T07:25:03,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741995_1171 (size=12629) 2024-11-27T07:25:03,555 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-27T07:25:03,558 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/b57b193b60ed416f9f021037a751afd1 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/b57b193b60ed416f9f021037a751afd1 2024-11-27T07:25:03,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:03,560 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 225833644b1bebd5fa53ae861d7c7070 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-27T07:25:03,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741996_1172 (size=4469) 2024-11-27T07:25:03,568 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 225833644b1bebd5fa53ae861d7c7070#A#compaction#141 average throughput is 0.33 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:25:03,569 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/dd8bba1c5e934853a2d2e53931ea71df is 175, key is test_row_0/A:col10/1732692301403/Put/seqid=0 2024-11-27T07:25:03,577 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 225833644b1bebd5fa53ae861d7c7070/B of 225833644b1bebd5fa53ae861d7c7070 into b57b193b60ed416f9f021037a751afd1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:25:03,577 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 225833644b1bebd5fa53ae861d7c7070: 2024-11-27T07:25:03,577 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., storeName=225833644b1bebd5fa53ae861d7c7070/B, priority=13, startTime=1732692303464; duration=0sec 2024-11-27T07:25:03,577 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:25:03,577 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 225833644b1bebd5fa53ae861d7c7070:B 2024-11-27T07:25:03,577 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:25:03,581 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:25:03,581 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 225833644b1bebd5fa53ae861d7c7070/C is initiating minor compaction (all files) 2024-11-27T07:25:03,582 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 225833644b1bebd5fa53ae861d7c7070/C in TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:03,582 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/42cce269fb91498abaedfffd0a9b0fc2, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/01330647c4d542798dcb9938ee6beac2, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/9e203d913fac497297d452a0c3efaef9] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp, totalSize=36.0 K 2024-11-27T07:25:03,582 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 42cce269fb91498abaedfffd0a9b0fc2, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732692299616 2024-11-27T07:25:03,583 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 01330647c4d542798dcb9938ee6beac2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732692300711 2024-11-27T07:25:03,583 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e203d913fac497297d452a0c3efaef9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732692301403 2024-11-27T07:25:03,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127ccba5d04f2024b558792120b72e0a4f0_225833644b1bebd5fa53ae861d7c7070 is 50, key is test_row_0/A:col10/1732692302624/Put/seqid=0 2024-11-27T07:25:03,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741997_1173 (size=31583) 2024-11-27T07:25:03,606 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 225833644b1bebd5fa53ae861d7c7070#C#compaction#144 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:25:03,607 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/43b4dc4492244102ac41bc13d5d2642f is 50, key is test_row_0/C:col10/1732692301403/Put/seqid=0 2024-11-27T07:25:03,609 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/dd8bba1c5e934853a2d2e53931ea71df as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/dd8bba1c5e934853a2d2e53931ea71df 2024-11-27T07:25:03,619 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 225833644b1bebd5fa53ae861d7c7070/A of 225833644b1bebd5fa53ae861d7c7070 into dd8bba1c5e934853a2d2e53931ea71df(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:25:03,619 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 225833644b1bebd5fa53ae861d7c7070: 2024-11-27T07:25:03,619 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., storeName=225833644b1bebd5fa53ae861d7c7070/A, priority=13, startTime=1732692303463; duration=0sec 2024-11-27T07:25:03,619 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:25:03,620 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 225833644b1bebd5fa53ae861d7c7070:A 2024-11-27T07:25:03,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741999_1175 (size=12629) 2024-11-27T07:25:03,662 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/43b4dc4492244102ac41bc13d5d2642f as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/43b4dc4492244102ac41bc13d5d2642f 2024-11-27T07:25:03,668 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 225833644b1bebd5fa53ae861d7c7070/C of 225833644b1bebd5fa53ae861d7c7070 into 43b4dc4492244102ac41bc13d5d2642f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:25:03,668 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 225833644b1bebd5fa53ae861d7c7070: 2024-11-27T07:25:03,668 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., storeName=225833644b1bebd5fa53ae861d7c7070/C, priority=13, startTime=1732692303471; duration=0sec 2024-11-27T07:25:03,668 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:25:03,668 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 225833644b1bebd5fa53ae861d7c7070:C 2024-11-27T07:25:03,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741998_1174 (size=12304) 2024-11-27T07:25:03,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:25:03,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. as already flushing 2024-11-27T07:25:03,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:03,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36314 deadline: 1732692363806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:03,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:03,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:03,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:03,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36312 deadline: 1732692363807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:03,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:03,811 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:03,811 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:03,811 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:03,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:03,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36298 deadline: 1732692363810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:03,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:03,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36352 deadline: 1732692363809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:03,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:03,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36338 deadline: 1732692363811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:03,812 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 289954c9202d,42117,1732692259671 for 'test_row_1' in 225833644b1bebd5fa53ae861d7c7070 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T07:25:03,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:03,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:03,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:03,813 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:03,813 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:03,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:03,814 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:03,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:03,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:03,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-27T07:25:03,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:03,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36314 deadline: 1732692363919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:03,923 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:03,923 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:03,923 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:03,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:03,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36298 deadline: 1732692363920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:03,928 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:03,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:03,928 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:03,928 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:03,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36338 deadline: 1732692363920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:03,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:03,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:03,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:03,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:03,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36312 deadline: 1732692363920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:03,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:03,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:03,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:04,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:25:04,076 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127ccba5d04f2024b558792120b72e0a4f0_225833644b1bebd5fa53ae861d7c7070 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127ccba5d04f2024b558792120b72e0a4f0_225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:25:04,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/0fe3d1c4379f4149a21e052bffdf3c67, store: [table=TestAcidGuarantees family=A region=225833644b1bebd5fa53ae861d7c7070] 2024-11-27T07:25:04,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/0fe3d1c4379f4149a21e052bffdf3c67 is 175, key is test_row_0/A:col10/1732692302624/Put/seqid=0 2024-11-27T07:25:04,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742000_1176 (size=31105) 2024-11-27T07:25:04,094 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=234, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/0fe3d1c4379f4149a21e052bffdf3c67 2024-11-27T07:25:04,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/304326091b0d4870973e679e31ec4a81 is 50, key is test_row_0/B:col10/1732692302624/Put/seqid=0 2024-11-27T07:25:04,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742001_1177 (size=12151) 2024-11-27T07:25:04,111 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/304326091b0d4870973e679e31ec4a81 2024-11-27T07:25:04,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/3c3aec40063e413ca89899d65cf216c4 is 50, key is test_row_0/C:col10/1732692302624/Put/seqid=0 2024-11-27T07:25:04,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:04,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36314 deadline: 1732692364128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:04,132 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:04,132 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:04,132 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:04,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742002_1178 (size=12151) 2024-11-27T07:25:04,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:04,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:04,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36338 deadline: 1732692364139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:04,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36298 deadline: 1732692364139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:04,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:04,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36312 deadline: 1732692364139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:04,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:04,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:04,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:04,140 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:04,140 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:04,140 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:04,140 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:04,140 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:04,140 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:04,420 INFO [master/289954c9202d:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-27T07:25:04,420 INFO [master/289954c9202d:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-27T07:25:04,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:04,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36314 deadline: 1732692364439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:04,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:04,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:04,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:04,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:04,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36338 deadline: 1732692364449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:04,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:04,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:04,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36312 deadline: 1732692364449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:04,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:04,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:04,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:04,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:04,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:04,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:04,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36298 deadline: 1732692364450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:04,452 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:04,452 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:04,452 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:04,534 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/3c3aec40063e413ca89899d65cf216c4 2024-11-27T07:25:04,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/0fe3d1c4379f4149a21e052bffdf3c67 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/0fe3d1c4379f4149a21e052bffdf3c67 2024-11-27T07:25:04,543 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/0fe3d1c4379f4149a21e052bffdf3c67, entries=150, sequenceid=234, filesize=30.4 K 2024-11-27T07:25:04,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/304326091b0d4870973e679e31ec4a81 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/304326091b0d4870973e679e31ec4a81 2024-11-27T07:25:04,549 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/304326091b0d4870973e679e31ec4a81, entries=150, sequenceid=234, filesize=11.9 K 2024-11-27T07:25:04,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/3c3aec40063e413ca89899d65cf216c4 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/3c3aec40063e413ca89899d65cf216c4 2024-11-27T07:25:04,555 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/3c3aec40063e413ca89899d65cf216c4, entries=150, sequenceid=234, filesize=11.9 K 2024-11-27T07:25:04,555 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 225833644b1bebd5fa53ae861d7c7070 in 995ms, sequenceid=234, compaction requested=false 2024-11-27T07:25:04,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 225833644b1bebd5fa53ae861d7c7070: 2024-11-27T07:25:04,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:04,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-27T07:25:04,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-27T07:25:04,559 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-27T07:25:04,559 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7750 sec 2024-11-27T07:25:04,560 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.7790 sec 2024-11-27T07:25:04,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-27T07:25:04,930 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T07:25:04,930 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T07:25:04,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-27T07:25:04,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-27T07:25:04,932 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T07:25:04,933 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T07:25:04,933 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T07:25:04,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:25:04,950 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T07:25:04,951 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 225833644b1bebd5fa53ae861d7c7070 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-27T07:25:04,967 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112787115b489906465ba5448fa594c62530_225833644b1bebd5fa53ae861d7c7070 is 50, key is test_row_0/A:col10/1732692303797/Put/seqid=0 2024-11-27T07:25:04,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742003_1179 (size=14794) 2024-11-27T07:25:04,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:04,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36314 deadline: 1732692364983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:04,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:04,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:04,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36312 deadline: 1732692364985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:04,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:04,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:04,989 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:04,989 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:04,989 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:04,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:04,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36298 deadline: 1732692364986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:04,994 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:04,994 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:04,994 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:04,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:04,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36338 deadline: 1732692364989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:04,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:04,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:04,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:05,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-27T07:25:05,086 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-27T07:25:05,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:05,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. as already flushing 2024-11-27T07:25:05,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:05,086 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:05,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:05,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:05,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:05,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36314 deadline: 1732692365099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:05,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:05,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:05,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36312 deadline: 1732692365099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:05,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:05,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:05,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36298 deadline: 1732692365099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:05,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:05,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:05,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36338 deadline: 1732692365099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:05,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:05,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:05,102 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:05,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:05,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:05,102 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:05,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:05,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:05,102 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:05,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-27T07:25:05,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:05,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. as already flushing 2024-11-27T07:25:05,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:05,239 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:05,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:05,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:05,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-27T07:25:05,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:05,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36314 deadline: 1732692365309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:05,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:05,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:05,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:05,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:05,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36312 deadline: 1732692365309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:05,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:05,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:05,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:05,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:05,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36338 deadline: 1732692365309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:05,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:05,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36298 deadline: 1732692365310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:05,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:05,314 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:05,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:05,314 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:05,314 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:05,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:05,373 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:25:05,377 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112787115b489906465ba5448fa594c62530_225833644b1bebd5fa53ae861d7c7070 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112787115b489906465ba5448fa594c62530_225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:25:05,378 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/553770897dfa44efaa9376f7cd79b6f7, store: [table=TestAcidGuarantees family=A region=225833644b1bebd5fa53ae861d7c7070] 2024-11-27T07:25:05,379 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/553770897dfa44efaa9376f7cd79b6f7 is 175, key is test_row_0/A:col10/1732692303797/Put/seqid=0 2024-11-27T07:25:05,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742004_1180 (size=39749) 2024-11-27T07:25:05,384 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=252, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/553770897dfa44efaa9376f7cd79b6f7 2024-11-27T07:25:05,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-27T07:25:05,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:05,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. as already flushing 2024-11-27T07:25:05,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:05,392 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:05,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:05,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:05,393 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/04fd0fd79e844c938ed7eb8b75582ad0 is 50, key is test_row_0/B:col10/1732692303797/Put/seqid=0 2024-11-27T07:25:05,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742005_1181 (size=12151) 2024-11-27T07:25:05,429 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/04fd0fd79e844c938ed7eb8b75582ad0 2024-11-27T07:25:05,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/c24a3590804a4e26a9ac69becac9c188 is 50, key is test_row_0/C:col10/1732692303797/Put/seqid=0 2024-11-27T07:25:05,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742006_1182 (size=12151) 2024-11-27T07:25:05,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-27T07:25:05,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:05,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. as already flushing 2024-11-27T07:25:05,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:05,546 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:05,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:05,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:05,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-27T07:25:05,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:05,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36298 deadline: 1732692365619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:05,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:05,621 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:05,621 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:05,621 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:05,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36314 deadline: 1732692365619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:05,621 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:05,621 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:05,621 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:05,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:05,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36312 deadline: 1732692365619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:05,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:05,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:05,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:05,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:05,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36338 deadline: 1732692365619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:05,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:05,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:05,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:05,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-27T07:25:05,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:05,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. as already flushing 2024-11-27T07:25:05,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:05,699 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:05,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:05,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:05,854 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-27T07:25:05,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:05,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. as already flushing 2024-11-27T07:25:05,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:05,855 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:05,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:05,855 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/c24a3590804a4e26a9ac69becac9c188 2024-11-27T07:25:05,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:05,866 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/553770897dfa44efaa9376f7cd79b6f7 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/553770897dfa44efaa9376f7cd79b6f7 2024-11-27T07:25:05,871 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/553770897dfa44efaa9376f7cd79b6f7, entries=200, sequenceid=252, filesize=38.8 K 2024-11-27T07:25:05,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/04fd0fd79e844c938ed7eb8b75582ad0 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/04fd0fd79e844c938ed7eb8b75582ad0 2024-11-27T07:25:05,880 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/04fd0fd79e844c938ed7eb8b75582ad0, entries=150, sequenceid=252, filesize=11.9 K 2024-11-27T07:25:05,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/c24a3590804a4e26a9ac69becac9c188 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/c24a3590804a4e26a9ac69becac9c188 2024-11-27T07:25:05,890 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/c24a3590804a4e26a9ac69becac9c188, entries=150, sequenceid=252, filesize=11.9 K 2024-11-27T07:25:05,891 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for 225833644b1bebd5fa53ae861d7c7070 in 941ms, sequenceid=252, compaction requested=true 2024-11-27T07:25:05,891 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 225833644b1bebd5fa53ae861d7c7070: 2024-11-27T07:25:05,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 225833644b1bebd5fa53ae861d7c7070:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T07:25:05,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:25:05,891 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:25:05,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 225833644b1bebd5fa53ae861d7c7070:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T07:25:05,891 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:25:05,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:25:05,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 225833644b1bebd5fa53ae861d7c7070:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T07:25:05,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:25:05,893 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102437 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:25:05,893 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:25:05,893 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1541): 225833644b1bebd5fa53ae861d7c7070/A is initiating minor compaction (all files) 2024-11-27T07:25:05,893 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 225833644b1bebd5fa53ae861d7c7070/B is initiating minor compaction (all files) 2024-11-27T07:25:05,893 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 225833644b1bebd5fa53ae861d7c7070/B in TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:05,893 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 225833644b1bebd5fa53ae861d7c7070/A in TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:05,893 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/dd8bba1c5e934853a2d2e53931ea71df, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/0fe3d1c4379f4149a21e052bffdf3c67, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/553770897dfa44efaa9376f7cd79b6f7] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp, totalSize=100.0 K 2024-11-27T07:25:05,893 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:05,893 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. files: [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/dd8bba1c5e934853a2d2e53931ea71df, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/0fe3d1c4379f4149a21e052bffdf3c67, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/553770897dfa44efaa9376f7cd79b6f7] 2024-11-27T07:25:05,894 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/b57b193b60ed416f9f021037a751afd1, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/304326091b0d4870973e679e31ec4a81, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/04fd0fd79e844c938ed7eb8b75582ad0] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp, totalSize=36.1 K 2024-11-27T07:25:05,895 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd8bba1c5e934853a2d2e53931ea71df, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732692301403 2024-11-27T07:25:05,895 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting b57b193b60ed416f9f021037a751afd1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732692301403 2024-11-27T07:25:05,895 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0fe3d1c4379f4149a21e052bffdf3c67, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732692302608 2024-11-27T07:25:05,895 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 304326091b0d4870973e679e31ec4a81, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732692302608 2024-11-27T07:25:05,895 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 04fd0fd79e844c938ed7eb8b75582ad0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732692303797 2024-11-27T07:25:05,895 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 553770897dfa44efaa9376f7cd79b6f7, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732692303797 2024-11-27T07:25:05,911 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=225833644b1bebd5fa53ae861d7c7070] 2024-11-27T07:25:05,927 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 225833644b1bebd5fa53ae861d7c7070#B#compaction#151 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:25:05,928 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/a9371fd13d2844148a1c98a03e12175c is 50, key is test_row_0/B:col10/1732692303797/Put/seqid=0 2024-11-27T07:25:05,945 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127808014e647c44adaadfe81fb0d5e9768_225833644b1bebd5fa53ae861d7c7070 store=[table=TestAcidGuarantees family=A region=225833644b1bebd5fa53ae861d7c7070] 2024-11-27T07:25:05,947 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127808014e647c44adaadfe81fb0d5e9768_225833644b1bebd5fa53ae861d7c7070, store=[table=TestAcidGuarantees family=A region=225833644b1bebd5fa53ae861d7c7070] 2024-11-27T07:25:05,947 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127808014e647c44adaadfe81fb0d5e9768_225833644b1bebd5fa53ae861d7c7070 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=225833644b1bebd5fa53ae861d7c7070] 2024-11-27T07:25:05,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742007_1183 (size=12731) 2024-11-27T07:25:05,967 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/a9371fd13d2844148a1c98a03e12175c as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/a9371fd13d2844148a1c98a03e12175c 2024-11-27T07:25:05,975 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 225833644b1bebd5fa53ae861d7c7070/B of 225833644b1bebd5fa53ae861d7c7070 into a9371fd13d2844148a1c98a03e12175c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:25:05,976 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 225833644b1bebd5fa53ae861d7c7070: 2024-11-27T07:25:05,976 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., storeName=225833644b1bebd5fa53ae861d7c7070/B, priority=13, startTime=1732692305891; duration=0sec 2024-11-27T07:25:05,976 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:25:05,976 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 225833644b1bebd5fa53ae861d7c7070:B 2024-11-27T07:25:05,976 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:25:05,977 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:25:05,977 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 225833644b1bebd5fa53ae861d7c7070/C is initiating minor compaction (all files) 2024-11-27T07:25:05,977 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 225833644b1bebd5fa53ae861d7c7070/C in TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:05,977 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/43b4dc4492244102ac41bc13d5d2642f, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/3c3aec40063e413ca89899d65cf216c4, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/c24a3590804a4e26a9ac69becac9c188] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp, totalSize=36.1 K 2024-11-27T07:25:05,979 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 43b4dc4492244102ac41bc13d5d2642f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732692301403 2024-11-27T07:25:05,980 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c3aec40063e413ca89899d65cf216c4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732692302608 2024-11-27T07:25:05,980 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting c24a3590804a4e26a9ac69becac9c188, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732692303797 2024-11-27T07:25:06,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742008_1184 (size=4469) 2024-11-27T07:25:06,006 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 225833644b1bebd5fa53ae861d7c7070#A#compaction#150 average throughput is 0.26 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:25:06,006 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/44d0dabb8cc64cd6801a6062ad26dbca is 175, key is test_row_0/A:col10/1732692303797/Put/seqid=0 2024-11-27T07:25:06,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-27T07:25:06,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:06,009 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 225833644b1bebd5fa53ae861d7c7070 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-27T07:25:06,019 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 225833644b1bebd5fa53ae861d7c7070#C#compaction#152 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:25:06,019 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/d816250e37154816a59d15432331e197 is 50, key is test_row_0/C:col10/1732692303797/Put/seqid=0 2024-11-27T07:25:06,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127a6aaf13a5e09436884f4be45804f645e_225833644b1bebd5fa53ae861d7c7070 is 50, key is test_row_0/A:col10/1732692304984/Put/seqid=0 2024-11-27T07:25:06,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742009_1185 (size=31685) 2024-11-27T07:25:06,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-27T07:25:06,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742011_1187 (size=12454) 2024-11-27T07:25:06,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742010_1186 (size=12731) 2024-11-27T07:25:06,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:25:06,089 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127a6aaf13a5e09436884f4be45804f645e_225833644b1bebd5fa53ae861d7c7070 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127a6aaf13a5e09436884f4be45804f645e_225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:25:06,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/8afef07fb8734e938c86b4614afa0fec, store: [table=TestAcidGuarantees family=A region=225833644b1bebd5fa53ae861d7c7070] 2024-11-27T07:25:06,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/8afef07fb8734e938c86b4614afa0fec is 175, key is test_row_0/A:col10/1732692304984/Put/seqid=0 2024-11-27T07:25:06,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742012_1188 (size=31255) 2024-11-27T07:25:06,132 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. as already flushing 2024-11-27T07:25:06,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:25:06,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:06,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36298 deadline: 1732692366144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:06,148 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:06,148 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:06,148 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:06,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:06,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36314 deadline: 1732692366145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:06,153 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:06,153 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:06,153 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:06,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:06,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36338 deadline: 1732692366147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:06,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:06,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:06,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:06,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36312 deadline: 1732692366147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:06,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:06,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:06,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:06,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:06,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:06,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36298 deadline: 1732692366259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:06,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:06,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36314 deadline: 1732692366259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:06,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:06,265 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:06,265 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:06,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36338 deadline: 1732692366259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:06,265 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:06,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:06,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:06,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:06,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36312 deadline: 1732692366259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:06,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:06,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:06,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:06,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:06,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:06,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:06,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:06,438 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/44d0dabb8cc64cd6801a6062ad26dbca as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/44d0dabb8cc64cd6801a6062ad26dbca 2024-11-27T07:25:06,444 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 225833644b1bebd5fa53ae861d7c7070/A of 225833644b1bebd5fa53ae861d7c7070 into 44d0dabb8cc64cd6801a6062ad26dbca(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:25:06,444 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 225833644b1bebd5fa53ae861d7c7070: 2024-11-27T07:25:06,445 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., storeName=225833644b1bebd5fa53ae861d7c7070/A, priority=13, startTime=1732692305891; duration=0sec 2024-11-27T07:25:06,445 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:25:06,445 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 225833644b1bebd5fa53ae861d7c7070:A 2024-11-27T07:25:06,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:06,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36298 deadline: 1732692366469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:06,474 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:06,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:06,474 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:06,474 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:06,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36314 deadline: 1732692366469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:06,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:06,474 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:06,474 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:06,474 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:06,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36338 deadline: 1732692366469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:06,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:06,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:06,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36312 deadline: 1732692366470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:06,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:06,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:06,475 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:06,475 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:06,475 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:06,488 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/d816250e37154816a59d15432331e197 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/d816250e37154816a59d15432331e197 2024-11-27T07:25:06,495 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 225833644b1bebd5fa53ae861d7c7070/C of 225833644b1bebd5fa53ae861d7c7070 into d816250e37154816a59d15432331e197(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:25:06,495 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 225833644b1bebd5fa53ae861d7c7070: 2024-11-27T07:25:06,495 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., storeName=225833644b1bebd5fa53ae861d7c7070/C, priority=13, startTime=1732692305891; duration=0sec 2024-11-27T07:25:06,495 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:25:06,495 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 225833644b1bebd5fa53ae861d7c7070:C 2024-11-27T07:25:06,497 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=275, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/8afef07fb8734e938c86b4614afa0fec 2024-11-27T07:25:06,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/b7ae8c95158b433796629687eaeb6b04 is 50, key is test_row_0/B:col10/1732692304984/Put/seqid=0 2024-11-27T07:25:06,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742013_1189 (size=12301) 2024-11-27T07:25:06,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:06,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36312 deadline: 1732692366779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:06,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:06,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:06,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:06,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:06,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36298 deadline: 1732692366779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:06,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:06,781 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:06,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36314 deadline: 1732692366780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:06,781 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:06,781 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:06,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:06,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:06,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:06,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:06,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36338 deadline: 1732692366780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:06,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:06,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:06,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:06,919 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/b7ae8c95158b433796629687eaeb6b04 2024-11-27T07:25:06,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/f4571adb23b84ef68edfc6a17a32d0f4 is 50, key is test_row_0/C:col10/1732692304984/Put/seqid=0 2024-11-27T07:25:06,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742014_1190 (size=12301) 2024-11-27T07:25:07,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-27T07:25:07,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:07,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:07,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36312 deadline: 1732692367289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:07,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36314 deadline: 1732692367289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:07,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:07,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:07,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:07,290 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:07,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:07,290 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:07,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:07,290 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:07,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36298 deadline: 1732692367289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:07,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36338 deadline: 1732692367289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:07,290 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:07,290 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:07,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:07,290 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:07,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:07,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:07,374 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/f4571adb23b84ef68edfc6a17a32d0f4 2024-11-27T07:25:07,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/8afef07fb8734e938c86b4614afa0fec as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/8afef07fb8734e938c86b4614afa0fec 2024-11-27T07:25:07,385 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/8afef07fb8734e938c86b4614afa0fec, entries=150, sequenceid=275, filesize=30.5 K 2024-11-27T07:25:07,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/b7ae8c95158b433796629687eaeb6b04 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/b7ae8c95158b433796629687eaeb6b04 2024-11-27T07:25:07,390 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/b7ae8c95158b433796629687eaeb6b04, entries=150, sequenceid=275, filesize=12.0 K 2024-11-27T07:25:07,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/f4571adb23b84ef68edfc6a17a32d0f4 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/f4571adb23b84ef68edfc6a17a32d0f4 2024-11-27T07:25:07,397 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/f4571adb23b84ef68edfc6a17a32d0f4, entries=150, sequenceid=275, filesize=12.0 K 2024-11-27T07:25:07,398 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for 225833644b1bebd5fa53ae861d7c7070 in 1388ms, sequenceid=275, compaction requested=false 2024-11-27T07:25:07,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 225833644b1bebd5fa53ae861d7c7070: 2024-11-27T07:25:07,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:07,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-27T07:25:07,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-27T07:25:07,401 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-27T07:25:07,401 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4660 sec 2024-11-27T07:25:07,403 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 2.4710 sec 2024-11-27T07:25:07,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:25:07,850 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T07:25:07,850 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 225833644b1bebd5fa53ae861d7c7070 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-27T07:25:07,857 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127cf61ce19df5e46848f429304f6fab0b3_225833644b1bebd5fa53ae861d7c7070 is 50, key is test_row_0/A:col10/1732692306143/Put/seqid=0 2024-11-27T07:25:07,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742015_1191 (size=14994) 2024-11-27T07:25:07,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:07,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36352 deadline: 1732692367908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:07,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:07,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:07,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:08,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:08,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36352 deadline: 1732692368019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:08,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:08,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:08,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:08,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:08,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36352 deadline: 1732692368229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:08,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:08,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:08,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:08,276 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:25:08,283 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127cf61ce19df5e46848f429304f6fab0b3_225833644b1bebd5fa53ae861d7c7070 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127cf61ce19df5e46848f429304f6fab0b3_225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:25:08,284 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/2061d197b8d8410f9f44a727cdf142e8, store: [table=TestAcidGuarantees family=A region=225833644b1bebd5fa53ae861d7c7070] 2024-11-27T07:25:08,285 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/2061d197b8d8410f9f44a727cdf142e8 is 175, key is test_row_0/A:col10/1732692306143/Put/seqid=0 2024-11-27T07:25:08,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742016_1192 (size=39949) 2024-11-27T07:25:08,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:08,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36312 deadline: 1732692368299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:08,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:08,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36298 deadline: 1732692368299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:08,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:08,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36338 deadline: 1732692368299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:08,299 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 289954c9202d,42117,1732692259671 for 'test_row_1' in 225833644b1bebd5fa53ae861d7c7070 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T07:25:08,300 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:08,300 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:08,300 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 289954c9202d,42117,1732692259671 for 'test_row_0' in 225833644b1bebd5fa53ae861d7c7070 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T07:25:08,300 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:08,300 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:08,300 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:08,300 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:08,300 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 289954c9202d,42117,1732692259671 for 'test_row_1' in 225833644b1bebd5fa53ae861d7c7070 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T07:25:08,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:08,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:08,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:08,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:08,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36314 deadline: 1732692368309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:08,311 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 289954c9202d,42117,1732692259671 for 'test_row_0' in 225833644b1bebd5fa53ae861d7c7070 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T07:25:08,312 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:08,312 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:08,312 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:08,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:08,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36352 deadline: 1732692368539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:08,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:08,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:08,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:08,698 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=293, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/2061d197b8d8410f9f44a727cdf142e8 2024-11-27T07:25:08,706 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/9aaa23cfdbfd4fffbb58a175c305adf6 is 50, key is test_row_0/B:col10/1732692306143/Put/seqid=0 2024-11-27T07:25:08,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742017_1193 (size=12301) 2024-11-27T07:25:08,720 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/9aaa23cfdbfd4fffbb58a175c305adf6 2024-11-27T07:25:08,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/fd0ee664bedd4551b9704c615c1efd9c is 50, key is test_row_0/C:col10/1732692306143/Put/seqid=0 2024-11-27T07:25:08,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742018_1194 (size=12301) 2024-11-27T07:25:09,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:09,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36352 deadline: 1732692369049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:09,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:09,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:09,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:09,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-27T07:25:09,110 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T07:25:09,110 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T07:25:09,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-27T07:25:09,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-27T07:25:09,112 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T07:25:09,113 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T07:25:09,113 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T07:25:09,132 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/fd0ee664bedd4551b9704c615c1efd9c 2024-11-27T07:25:09,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/2061d197b8d8410f9f44a727cdf142e8 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/2061d197b8d8410f9f44a727cdf142e8 2024-11-27T07:25:09,141 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/2061d197b8d8410f9f44a727cdf142e8, entries=200, sequenceid=293, filesize=39.0 K 2024-11-27T07:25:09,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/9aaa23cfdbfd4fffbb58a175c305adf6 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/9aaa23cfdbfd4fffbb58a175c305adf6 2024-11-27T07:25:09,146 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/9aaa23cfdbfd4fffbb58a175c305adf6, entries=150, sequenceid=293, filesize=12.0 K 2024-11-27T07:25:09,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/fd0ee664bedd4551b9704c615c1efd9c as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/fd0ee664bedd4551b9704c615c1efd9c 2024-11-27T07:25:09,150 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/fd0ee664bedd4551b9704c615c1efd9c, entries=150, sequenceid=293, filesize=12.0 K 2024-11-27T07:25:09,151 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 225833644b1bebd5fa53ae861d7c7070 in 1301ms, sequenceid=293, compaction requested=true 2024-11-27T07:25:09,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 225833644b1bebd5fa53ae861d7c7070: 2024-11-27T07:25:09,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 225833644b1bebd5fa53ae861d7c7070:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T07:25:09,152 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:25:09,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:25:09,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 225833644b1bebd5fa53ae861d7c7070:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T07:25:09,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:25:09,152 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:25:09,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 225833644b1bebd5fa53ae861d7c7070:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T07:25:09,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:25:09,152 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:25:09,152 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102889 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:25:09,153 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 225833644b1bebd5fa53ae861d7c7070/B is initiating minor compaction (all files) 2024-11-27T07:25:09,153 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1541): 225833644b1bebd5fa53ae861d7c7070/A is initiating minor compaction (all files) 2024-11-27T07:25:09,153 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 225833644b1bebd5fa53ae861d7c7070/B in TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:09,153 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 225833644b1bebd5fa53ae861d7c7070/A in TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:09,153 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/a9371fd13d2844148a1c98a03e12175c, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/b7ae8c95158b433796629687eaeb6b04, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/9aaa23cfdbfd4fffbb58a175c305adf6] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp, totalSize=36.5 K 2024-11-27T07:25:09,153 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/44d0dabb8cc64cd6801a6062ad26dbca, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/8afef07fb8734e938c86b4614afa0fec, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/2061d197b8d8410f9f44a727cdf142e8] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp, totalSize=100.5 K 2024-11-27T07:25:09,153 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:09,153 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. files: [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/44d0dabb8cc64cd6801a6062ad26dbca, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/8afef07fb8734e938c86b4614afa0fec, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/2061d197b8d8410f9f44a727cdf142e8] 2024-11-27T07:25:09,153 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44d0dabb8cc64cd6801a6062ad26dbca, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732692303797 2024-11-27T07:25:09,153 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting a9371fd13d2844148a1c98a03e12175c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732692303797 2024-11-27T07:25:09,154 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8afef07fb8734e938c86b4614afa0fec, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732692304983 2024-11-27T07:25:09,154 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting b7ae8c95158b433796629687eaeb6b04, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732692304983 2024-11-27T07:25:09,154 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2061d197b8d8410f9f44a727cdf142e8, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732692306134 2024-11-27T07:25:09,154 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 9aaa23cfdbfd4fffbb58a175c305adf6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732692306134 2024-11-27T07:25:09,160 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=225833644b1bebd5fa53ae861d7c7070] 2024-11-27T07:25:09,162 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 225833644b1bebd5fa53ae861d7c7070#B#compaction#160 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:25:09,162 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/0b21f98c58b84dd8bdc3211156783450 is 50, key is test_row_0/B:col10/1732692306143/Put/seqid=0 2024-11-27T07:25:09,166 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127e206993972f04258a2473af748f6d060_225833644b1bebd5fa53ae861d7c7070 store=[table=TestAcidGuarantees family=A region=225833644b1bebd5fa53ae861d7c7070] 2024-11-27T07:25:09,168 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127e206993972f04258a2473af748f6d060_225833644b1bebd5fa53ae861d7c7070, store=[table=TestAcidGuarantees family=A region=225833644b1bebd5fa53ae861d7c7070] 2024-11-27T07:25:09,168 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127e206993972f04258a2473af748f6d060_225833644b1bebd5fa53ae861d7c7070 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=225833644b1bebd5fa53ae861d7c7070] 2024-11-27T07:25:09,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742019_1195 (size=12983) 2024-11-27T07:25:09,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742020_1196 (size=4469) 2024-11-27T07:25:09,180 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 225833644b1bebd5fa53ae861d7c7070#A#compaction#159 average throughput is 1.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:25:09,180 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/41bcc16a36db4e8f8895fc9ed27334d9 is 175, key is test_row_0/A:col10/1732692306143/Put/seqid=0 2024-11-27T07:25:09,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742021_1197 (size=31937) 2024-11-27T07:25:09,211 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/41bcc16a36db4e8f8895fc9ed27334d9 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/41bcc16a36db4e8f8895fc9ed27334d9 2024-11-27T07:25:09,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-27T07:25:09,223 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 225833644b1bebd5fa53ae861d7c7070/A of 225833644b1bebd5fa53ae861d7c7070 into 41bcc16a36db4e8f8895fc9ed27334d9(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:25:09,223 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 225833644b1bebd5fa53ae861d7c7070: 2024-11-27T07:25:09,224 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., storeName=225833644b1bebd5fa53ae861d7c7070/A, priority=13, startTime=1732692309151; duration=0sec 2024-11-27T07:25:09,224 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:25:09,224 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 225833644b1bebd5fa53ae861d7c7070:A 2024-11-27T07:25:09,224 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:25:09,227 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:25:09,227 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1541): 225833644b1bebd5fa53ae861d7c7070/C is initiating minor compaction (all files) 2024-11-27T07:25:09,228 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 225833644b1bebd5fa53ae861d7c7070/C in TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:09,228 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/d816250e37154816a59d15432331e197, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/f4571adb23b84ef68edfc6a17a32d0f4, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/fd0ee664bedd4551b9704c615c1efd9c] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp, totalSize=36.5 K 2024-11-27T07:25:09,229 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting d816250e37154816a59d15432331e197, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732692303797 2024-11-27T07:25:09,229 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4571adb23b84ef68edfc6a17a32d0f4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732692304983 2024-11-27T07:25:09,229 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd0ee664bedd4551b9704c615c1efd9c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732692306134 2024-11-27T07:25:09,241 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 225833644b1bebd5fa53ae861d7c7070#C#compaction#161 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:25:09,242 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/f7f6b4ed072c407c9ebb8cbe51de8525 is 50, key is test_row_0/C:col10/1732692306143/Put/seqid=0 2024-11-27T07:25:09,267 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-27T07:25:09,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:09,267 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 225833644b1bebd5fa53ae861d7c7070 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-27T07:25:09,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127f561096555b6484082bf4ad843a86c56_225833644b1bebd5fa53ae861d7c7070 is 50, key is test_row_0/A:col10/1732692307906/Put/seqid=0 2024-11-27T07:25:09,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742022_1198 (size=12983) 2024-11-27T07:25:09,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742023_1199 (size=12454) 2024-11-27T07:25:09,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-27T07:25:09,590 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/0b21f98c58b84dd8bdc3211156783450 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/0b21f98c58b84dd8bdc3211156783450 2024-11-27T07:25:09,607 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 225833644b1bebd5fa53ae861d7c7070/B of 225833644b1bebd5fa53ae861d7c7070 into 0b21f98c58b84dd8bdc3211156783450(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:25:09,607 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 225833644b1bebd5fa53ae861d7c7070: 2024-11-27T07:25:09,607 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., storeName=225833644b1bebd5fa53ae861d7c7070/B, priority=13, startTime=1732692309152; duration=0sec 2024-11-27T07:25:09,608 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:25:09,608 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 225833644b1bebd5fa53ae861d7c7070:B 2024-11-27T07:25:09,697 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/f7f6b4ed072c407c9ebb8cbe51de8525 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/f7f6b4ed072c407c9ebb8cbe51de8525 2024-11-27T07:25:09,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:25:09,707 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 225833644b1bebd5fa53ae861d7c7070/C of 225833644b1bebd5fa53ae861d7c7070 into f7f6b4ed072c407c9ebb8cbe51de8525(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:25:09,707 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 225833644b1bebd5fa53ae861d7c7070: 2024-11-27T07:25:09,707 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., storeName=225833644b1bebd5fa53ae861d7c7070/C, priority=13, startTime=1732692309152; duration=0sec 2024-11-27T07:25:09,707 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:25:09,707 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 225833644b1bebd5fa53ae861d7c7070:C 2024-11-27T07:25:09,712 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127f561096555b6484082bf4ad843a86c56_225833644b1bebd5fa53ae861d7c7070 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127f561096555b6484082bf4ad843a86c56_225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:25:09,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/9e301b2d30ed4a3ca98b2e7cf102b186, store: [table=TestAcidGuarantees family=A region=225833644b1bebd5fa53ae861d7c7070] 2024-11-27T07:25:09,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/9e301b2d30ed4a3ca98b2e7cf102b186 is 175, key is test_row_0/A:col10/1732692307906/Put/seqid=0 2024-11-27T07:25:09,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-27T07:25:09,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742024_1200 (size=31255) 2024-11-27T07:25:10,061 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. as already flushing 2024-11-27T07:25:10,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:25:10,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:10,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36352 deadline: 1732692370106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:10,109 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:10,109 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:10,109 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:10,156 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=314, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/9e301b2d30ed4a3ca98b2e7cf102b186 2024-11-27T07:25:10,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/067cfbc8e62d4ca4bbf8c276b5b83153 is 50, key is test_row_0/B:col10/1732692307906/Put/seqid=0 2024-11-27T07:25:10,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742025_1201 (size=12301) 2024-11-27T07:25:10,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:10,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36352 deadline: 1732692370219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:10,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:10,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:10,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:10,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-27T07:25:10,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:10,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36312 deadline: 1732692370309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:10,312 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 289954c9202d,42117,1732692259671 for 'test_row_1' in 225833644b1bebd5fa53ae861d7c7070 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T07:25:10,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:10,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:10,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:10,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:10,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36298 deadline: 1732692370319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:10,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:10,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36314 deadline: 1732692370319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:10,323 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 289954c9202d,42117,1732692259671 for 'test_row_0' in 225833644b1bebd5fa53ae861d7c7070 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T07:25:10,324 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 289954c9202d,42117,1732692259671 for 'test_row_0' in 225833644b1bebd5fa53ae861d7c7070 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T07:25:10,324 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:10,324 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:10,324 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:10,324 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:10,324 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:10,324 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:10,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:10,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36338 deadline: 1732692370329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:10,332 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 289954c9202d,42117,1732692259671 for 'test_row_1' in 225833644b1bebd5fa53ae861d7c7070 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T07:25:10,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:10,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:10,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:10,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:10,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36352 deadline: 1732692370428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:10,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:10,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:10,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:10,570 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/067cfbc8e62d4ca4bbf8c276b5b83153 2024-11-27T07:25:10,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/ae7260d3a3bc4d55ad3a81ad26e3790a is 50, key is test_row_0/C:col10/1732692307906/Put/seqid=0 2024-11-27T07:25:10,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742026_1202 (size=12301) 2024-11-27T07:25:10,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:10,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36352 deadline: 1732692370738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:10,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:10,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:10,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:10,988 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/ae7260d3a3bc4d55ad3a81ad26e3790a 2024-11-27T07:25:10,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/9e301b2d30ed4a3ca98b2e7cf102b186 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/9e301b2d30ed4a3ca98b2e7cf102b186 2024-11-27T07:25:10,996 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/9e301b2d30ed4a3ca98b2e7cf102b186, entries=150, sequenceid=314, filesize=30.5 K 2024-11-27T07:25:10,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/067cfbc8e62d4ca4bbf8c276b5b83153 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/067cfbc8e62d4ca4bbf8c276b5b83153 2024-11-27T07:25:11,003 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/067cfbc8e62d4ca4bbf8c276b5b83153, entries=150, sequenceid=314, filesize=12.0 K 2024-11-27T07:25:11,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/ae7260d3a3bc4d55ad3a81ad26e3790a as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/ae7260d3a3bc4d55ad3a81ad26e3790a 2024-11-27T07:25:11,008 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/ae7260d3a3bc4d55ad3a81ad26e3790a, entries=150, sequenceid=314, filesize=12.0 K 2024-11-27T07:25:11,009 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 225833644b1bebd5fa53ae861d7c7070 in 1742ms, sequenceid=314, compaction requested=false 2024-11-27T07:25:11,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 225833644b1bebd5fa53ae861d7c7070: 2024-11-27T07:25:11,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:11,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-27T07:25:11,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-27T07:25:11,013 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-27T07:25:11,013 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8980 sec 2024-11-27T07:25:11,015 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 1.9030 sec 2024-11-27T07:25:11,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:25:11,251 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T07:25:11,252 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 225833644b1bebd5fa53ae861d7c7070 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-27T07:25:11,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-27T07:25:11,259 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T07:25:11,260 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T07:25:11,260 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127769ba03a5429477694366d8bf519c59e_225833644b1bebd5fa53ae861d7c7070 is 50, key is test_row_0/A:col10/1732692310104/Put/seqid=0 2024-11-27T07:25:11,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-27T07:25:11,262 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T07:25:11,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-27T07:25:11,263 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T07:25:11,263 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T07:25:11,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742027_1203 (size=14994) 2024-11-27T07:25:11,268 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:25:11,273 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127769ba03a5429477694366d8bf519c59e_225833644b1bebd5fa53ae861d7c7070 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127769ba03a5429477694366d8bf519c59e_225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:25:11,274 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/bcc6d9140ed841be83f14ffe19cd9484, store: [table=TestAcidGuarantees family=A region=225833644b1bebd5fa53ae861d7c7070] 2024-11-27T07:25:11,275 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/bcc6d9140ed841be83f14ffe19cd9484 is 175, key is test_row_0/A:col10/1732692310104/Put/seqid=0 2024-11-27T07:25:11,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742028_1204 (size=39949) 2024-11-27T07:25:11,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:11,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36352 deadline: 1732692371323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:11,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:11,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:11,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:11,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-27T07:25:11,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-27T07:25:11,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:11,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. as already flushing 2024-11-27T07:25:11,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:11,416 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:11,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:11,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:11,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:11,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36352 deadline: 1732692371429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:11,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:11,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:11,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:11,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-27T07:25:11,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:11,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. as already flushing 2024-11-27T07:25:11,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:11,573 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:11,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:11,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:11,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-27T07:25:11,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:11,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36352 deadline: 1732692371639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:11,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:11,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:11,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:11,699 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=333, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/bcc6d9140ed841be83f14ffe19cd9484 2024-11-27T07:25:11,709 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/34d33d3d04d24b3ab243ed8ded171342 is 50, key is test_row_0/B:col10/1732692310104/Put/seqid=0 2024-11-27T07:25:11,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742029_1205 (size=12301) 2024-11-27T07:25:11,717 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/34d33d3d04d24b3ab243ed8ded171342 2024-11-27T07:25:11,726 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-27T07:25:11,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:11,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. as already flushing 2024-11-27T07:25:11,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:11,726 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:11,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:11,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:11,730 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/b4402022ac624c97a3a1576118541ebd is 50, key is test_row_0/C:col10/1732692310104/Put/seqid=0 2024-11-27T07:25:11,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742030_1206 (size=12301) 2024-11-27T07:25:11,879 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-27T07:25:11,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:11,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. as already flushing 2024-11-27T07:25:11,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:11,879 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:11,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:11,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:11,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-27T07:25:11,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:11,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36352 deadline: 1732692371949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:11,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:11,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:11,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:12,032 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-27T07:25:12,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:12,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. as already flushing 2024-11-27T07:25:12,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:12,032 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:12,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:12,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:12,134 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/b4402022ac624c97a3a1576118541ebd 2024-11-27T07:25:12,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/bcc6d9140ed841be83f14ffe19cd9484 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/bcc6d9140ed841be83f14ffe19cd9484 2024-11-27T07:25:12,150 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/bcc6d9140ed841be83f14ffe19cd9484, entries=200, sequenceid=333, filesize=39.0 K 2024-11-27T07:25:12,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/34d33d3d04d24b3ab243ed8ded171342 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/34d33d3d04d24b3ab243ed8ded171342 2024-11-27T07:25:12,157 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/34d33d3d04d24b3ab243ed8ded171342, entries=150, sequenceid=333, filesize=12.0 K 2024-11-27T07:25:12,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/b4402022ac624c97a3a1576118541ebd as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/b4402022ac624c97a3a1576118541ebd 2024-11-27T07:25:12,163 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/b4402022ac624c97a3a1576118541ebd, entries=150, sequenceid=333, filesize=12.0 K 2024-11-27T07:25:12,164 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 225833644b1bebd5fa53ae861d7c7070 in 913ms, sequenceid=333, compaction requested=true 2024-11-27T07:25:12,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 225833644b1bebd5fa53ae861d7c7070: 2024-11-27T07:25:12,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 225833644b1bebd5fa53ae861d7c7070:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T07:25:12,164 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:25:12,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:25:12,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 225833644b1bebd5fa53ae861d7c7070:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T07:25:12,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:25:12,164 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:25:12,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 225833644b1bebd5fa53ae861d7c7070:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T07:25:12,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:25:12,165 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:25:12,165 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:25:12,165 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1541): 225833644b1bebd5fa53ae861d7c7070/A is initiating minor compaction (all files) 2024-11-27T07:25:12,165 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 225833644b1bebd5fa53ae861d7c7070/B is initiating minor compaction (all files) 2024-11-27T07:25:12,165 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 225833644b1bebd5fa53ae861d7c7070/B in TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:12,165 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 225833644b1bebd5fa53ae861d7c7070/A in TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:12,165 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/0b21f98c58b84dd8bdc3211156783450, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/067cfbc8e62d4ca4bbf8c276b5b83153, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/34d33d3d04d24b3ab243ed8ded171342] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp, totalSize=36.7 K 2024-11-27T07:25:12,165 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/41bcc16a36db4e8f8895fc9ed27334d9, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/9e301b2d30ed4a3ca98b2e7cf102b186, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/bcc6d9140ed841be83f14ffe19cd9484] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp, totalSize=100.7 K 2024-11-27T07:25:12,165 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:12,166 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. files: [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/41bcc16a36db4e8f8895fc9ed27334d9, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/9e301b2d30ed4a3ca98b2e7cf102b186, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/bcc6d9140ed841be83f14ffe19cd9484] 2024-11-27T07:25:12,166 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b21f98c58b84dd8bdc3211156783450, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732692306134 2024-11-27T07:25:12,166 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 067cfbc8e62d4ca4bbf8c276b5b83153, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1732692307868 2024-11-27T07:25:12,166 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41bcc16a36db4e8f8895fc9ed27334d9, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732692306134 2024-11-27T07:25:12,167 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 34d33d3d04d24b3ab243ed8ded171342, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1732692310101 2024-11-27T07:25:12,167 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e301b2d30ed4a3ca98b2e7cf102b186, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1732692307868 2024-11-27T07:25:12,168 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting bcc6d9140ed841be83f14ffe19cd9484, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1732692310092 2024-11-27T07:25:12,181 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=225833644b1bebd5fa53ae861d7c7070] 2024-11-27T07:25:12,182 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 225833644b1bebd5fa53ae861d7c7070#B#compaction#168 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:25:12,183 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/2013697e01e247988309d7b18630425b is 50, key is test_row_0/B:col10/1732692310104/Put/seqid=0 2024-11-27T07:25:12,184 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127c84530d71b934992abf5f74a28536f80_225833644b1bebd5fa53ae861d7c7070 store=[table=TestAcidGuarantees family=A region=225833644b1bebd5fa53ae861d7c7070] 2024-11-27T07:25:12,185 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-27T07:25:12,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:12,185 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing 225833644b1bebd5fa53ae861d7c7070 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-27T07:25:12,186 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127c84530d71b934992abf5f74a28536f80_225833644b1bebd5fa53ae861d7c7070, store=[table=TestAcidGuarantees family=A region=225833644b1bebd5fa53ae861d7c7070] 2024-11-27T07:25:12,186 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127c84530d71b934992abf5f74a28536f80_225833644b1bebd5fa53ae861d7c7070 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=225833644b1bebd5fa53ae861d7c7070] 2024-11-27T07:25:12,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742031_1207 (size=13085) 2024-11-27T07:25:12,221 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/2013697e01e247988309d7b18630425b as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/2013697e01e247988309d7b18630425b 2024-11-27T07:25:12,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742032_1208 (size=4469) 2024-11-27T07:25:12,228 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 225833644b1bebd5fa53ae861d7c7070#A#compaction#169 average throughput is 0.52 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:25:12,229 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/694136cdd3cd4dd2b0373cc16b7ce862 is 175, key is test_row_0/A:col10/1732692310104/Put/seqid=0 2024-11-27T07:25:12,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112790837c8c25234d68988b53700547e560_225833644b1bebd5fa53ae861d7c7070 is 50, key is test_row_0/A:col10/1732692311322/Put/seqid=0 2024-11-27T07:25:12,232 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 225833644b1bebd5fa53ae861d7c7070/B of 225833644b1bebd5fa53ae861d7c7070 into 2013697e01e247988309d7b18630425b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:25:12,233 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 225833644b1bebd5fa53ae861d7c7070: 2024-11-27T07:25:12,233 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., storeName=225833644b1bebd5fa53ae861d7c7070/B, priority=13, startTime=1732692312164; duration=0sec 2024-11-27T07:25:12,233 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:25:12,233 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 225833644b1bebd5fa53ae861d7c7070:B 2024-11-27T07:25:12,233 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:25:12,234 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:25:12,234 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 225833644b1bebd5fa53ae861d7c7070/C is initiating minor compaction (all files) 2024-11-27T07:25:12,234 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 225833644b1bebd5fa53ae861d7c7070/C in TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:12,234 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/f7f6b4ed072c407c9ebb8cbe51de8525, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/ae7260d3a3bc4d55ad3a81ad26e3790a, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/b4402022ac624c97a3a1576118541ebd] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp, totalSize=36.7 K 2024-11-27T07:25:12,235 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting f7f6b4ed072c407c9ebb8cbe51de8525, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732692306134 2024-11-27T07:25:12,235 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting ae7260d3a3bc4d55ad3a81ad26e3790a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1732692307868 2024-11-27T07:25:12,235 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting b4402022ac624c97a3a1576118541ebd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1732692310101 2024-11-27T07:25:12,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742033_1209 (size=32039) 2024-11-27T07:25:12,246 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 225833644b1bebd5fa53ae861d7c7070#C#compaction#171 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:25:12,247 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/8ea0713f77d3410e8710bd632ded4037 is 50, key is test_row_0/C:col10/1732692310104/Put/seqid=0 2024-11-27T07:25:12,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742034_1210 (size=12454) 2024-11-27T07:25:12,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742035_1211 (size=13085) 2024-11-27T07:25:12,303 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/8ea0713f77d3410e8710bd632ded4037 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/8ea0713f77d3410e8710bd632ded4037 2024-11-27T07:25:12,309 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 225833644b1bebd5fa53ae861d7c7070/C of 225833644b1bebd5fa53ae861d7c7070 into 8ea0713f77d3410e8710bd632ded4037(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:25:12,309 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 225833644b1bebd5fa53ae861d7c7070: 2024-11-27T07:25:12,309 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., storeName=225833644b1bebd5fa53ae861d7c7070/C, priority=13, startTime=1732692312164; duration=0sec 2024-11-27T07:25:12,309 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:25:12,310 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 225833644b1bebd5fa53ae861d7c7070:C 2024-11-27T07:25:12,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-27T07:25:12,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:25:12,460 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. as already flushing 2024-11-27T07:25:12,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:12,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36352 deadline: 1732692372509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:12,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:12,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:12,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:12,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:12,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36352 deadline: 1732692372619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:12,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:12,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:12,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:12,653 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/694136cdd3cd4dd2b0373cc16b7ce862 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/694136cdd3cd4dd2b0373cc16b7ce862 2024-11-27T07:25:12,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:25:12,661 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 225833644b1bebd5fa53ae861d7c7070/A of 225833644b1bebd5fa53ae861d7c7070 into 694136cdd3cd4dd2b0373cc16b7ce862(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:25:12,661 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 225833644b1bebd5fa53ae861d7c7070: 2024-11-27T07:25:12,661 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., storeName=225833644b1bebd5fa53ae861d7c7070/A, priority=13, startTime=1732692312164; duration=0sec 2024-11-27T07:25:12,662 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:25:12,662 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 225833644b1bebd5fa53ae861d7c7070:A 2024-11-27T07:25:12,663 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112790837c8c25234d68988b53700547e560_225833644b1bebd5fa53ae861d7c7070 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112790837c8c25234d68988b53700547e560_225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:25:12,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/703d607657dd44668c21182a1f58d988, store: [table=TestAcidGuarantees family=A region=225833644b1bebd5fa53ae861d7c7070] 2024-11-27T07:25:12,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/703d607657dd44668c21182a1f58d988 is 175, key is test_row_0/A:col10/1732692311322/Put/seqid=0 2024-11-27T07:25:12,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742036_1212 (size=31255) 2024-11-27T07:25:12,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:12,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36352 deadline: 1732692372839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:12,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:12,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:12,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:13,113 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=352, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/703d607657dd44668c21182a1f58d988 2024-11-27T07:25:13,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/c1baa9adb92b4694b53c0598fdf6183d is 50, key is test_row_0/B:col10/1732692311322/Put/seqid=0 2024-11-27T07:25:13,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742037_1213 (size=12301) 2024-11-27T07:25:13,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:13,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36352 deadline: 1732692373149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:13,152 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:13,152 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:13,152 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:13,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-27T07:25:13,547 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/c1baa9adb92b4694b53c0598fdf6183d 2024-11-27T07:25:13,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/6e09bf5c89c94b709bf5c434610a2858 is 50, key is test_row_0/C:col10/1732692311322/Put/seqid=0 2024-11-27T07:25:13,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742038_1214 (size=12301) 2024-11-27T07:25:13,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:13,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36352 deadline: 1732692373659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:13,662 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:13,662 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:13,662 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:13,958 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/6e09bf5c89c94b709bf5c434610a2858 2024-11-27T07:25:13,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/703d607657dd44668c21182a1f58d988 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/703d607657dd44668c21182a1f58d988 2024-11-27T07:25:13,969 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/703d607657dd44668c21182a1f58d988, entries=150, sequenceid=352, filesize=30.5 K 2024-11-27T07:25:13,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/c1baa9adb92b4694b53c0598fdf6183d as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/c1baa9adb92b4694b53c0598fdf6183d 2024-11-27T07:25:13,977 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/c1baa9adb92b4694b53c0598fdf6183d, entries=150, sequenceid=352, filesize=12.0 K 2024-11-27T07:25:13,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/6e09bf5c89c94b709bf5c434610a2858 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/6e09bf5c89c94b709bf5c434610a2858 2024-11-27T07:25:13,985 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/6e09bf5c89c94b709bf5c434610a2858, entries=150, sequenceid=352, filesize=12.0 K 2024-11-27T07:25:13,987 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 225833644b1bebd5fa53ae861d7c7070 in 1801ms, sequenceid=352, compaction requested=false 2024-11-27T07:25:13,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for 225833644b1bebd5fa53ae861d7c7070: 2024-11-27T07:25:13,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:13,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-27T07:25:13,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-27T07:25:13,994 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-27T07:25:13,994 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7290 sec 2024-11-27T07:25:13,995 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 2.7340 sec 2024-11-27T07:25:14,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:25:14,330 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T07:25:14,331 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 225833644b1bebd5fa53ae861d7c7070 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-27T07:25:14,360 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112765a674e8ff33452ca5d79688591138af_225833644b1bebd5fa53ae861d7c7070 is 50, key is test_row_0/A:col10/1732692312505/Put/seqid=0 2024-11-27T07:25:14,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:14,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36338 deadline: 1732692374361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:14,363 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 289954c9202d,42117,1732692259671 for 'test_row_1' in 225833644b1bebd5fa53ae861d7c7070 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T07:25:14,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:14,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:14,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:14,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:14,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36314 deadline: 1732692374362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:14,365 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:14,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:14,366 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:14,366 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:14,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36312 deadline: 1732692374363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:14,366 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:14,366 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:14,366 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:14,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:14,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36298 deadline: 1732692374363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:14,367 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:14,367 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:14,367 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:14,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742039_1215 (size=14994) 2024-11-27T07:25:14,368 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:25:14,373 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112765a674e8ff33452ca5d79688591138af_225833644b1bebd5fa53ae861d7c7070 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112765a674e8ff33452ca5d79688591138af_225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:25:14,380 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/be8ec9a19b2042bf95b450e3818ed82a, store: [table=TestAcidGuarantees family=A region=225833644b1bebd5fa53ae861d7c7070] 2024-11-27T07:25:14,381 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/be8ec9a19b2042bf95b450e3818ed82a is 175, key is test_row_0/A:col10/1732692312505/Put/seqid=0 2024-11-27T07:25:14,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742040_1216 (size=39949) 2024-11-27T07:25:14,398 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=373, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/be8ec9a19b2042bf95b450e3818ed82a 2024-11-27T07:25:14,405 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/9af2d50204054a1e8d4f62791916fd9a is 50, key is test_row_0/B:col10/1732692312505/Put/seqid=0 2024-11-27T07:25:14,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742041_1217 (size=12301) 2024-11-27T07:25:14,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:14,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36314 deadline: 1732692374469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:14,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:14,472 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:14,472 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:14,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36312 deadline: 1732692374469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:14,472 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:14,472 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:14,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:14,472 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:14,472 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:14,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36298 deadline: 1732692374469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:14,473 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:14,473 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:14,473 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:14,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:14,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:14,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36352 deadline: 1732692374678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:14,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36314 deadline: 1732692374679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:14,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:14,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:14,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36298 deadline: 1732692374679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:14,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36312 deadline: 1732692374679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:14,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:14,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:14,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:14,682 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 289954c9202d,42117,1732692259671 for 'test_row_1' in 225833644b1bebd5fa53ae861d7c7070 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T07:25:14,682 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:14,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:14,682 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:14,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:14,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:14,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:14,682 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:14,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:14,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:14,811 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/9af2d50204054a1e8d4f62791916fd9a 2024-11-27T07:25:14,821 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/dfa9211a9323494999c9576c608d7074 is 50, key is test_row_0/C:col10/1732692312505/Put/seqid=0 2024-11-27T07:25:14,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742042_1218 (size=12301) 2024-11-27T07:25:14,825 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/dfa9211a9323494999c9576c608d7074 2024-11-27T07:25:14,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/be8ec9a19b2042bf95b450e3818ed82a as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/be8ec9a19b2042bf95b450e3818ed82a 2024-11-27T07:25:14,836 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/be8ec9a19b2042bf95b450e3818ed82a, entries=200, sequenceid=373, filesize=39.0 K 2024-11-27T07:25:14,838 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/9af2d50204054a1e8d4f62791916fd9a as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/9af2d50204054a1e8d4f62791916fd9a 2024-11-27T07:25:14,844 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/9af2d50204054a1e8d4f62791916fd9a, entries=150, sequenceid=373, filesize=12.0 K 2024-11-27T07:25:14,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/dfa9211a9323494999c9576c608d7074 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/dfa9211a9323494999c9576c608d7074 2024-11-27T07:25:14,852 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/dfa9211a9323494999c9576c608d7074, entries=150, sequenceid=373, filesize=12.0 K 2024-11-27T07:25:14,854 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 225833644b1bebd5fa53ae861d7c7070 in 524ms, sequenceid=373, compaction requested=true 2024-11-27T07:25:14,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 225833644b1bebd5fa53ae861d7c7070: 2024-11-27T07:25:14,854 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:25:14,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 225833644b1bebd5fa53ae861d7c7070:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T07:25:14,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:25:14,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 225833644b1bebd5fa53ae861d7c7070:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T07:25:14,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:25:14,855 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:25:14,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 225833644b1bebd5fa53ae861d7c7070:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T07:25:14,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:25:14,857 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103243 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:25:14,857 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 225833644b1bebd5fa53ae861d7c7070/A is initiating minor compaction (all files) 2024-11-27T07:25:14,857 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 225833644b1bebd5fa53ae861d7c7070/A in TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:14,857 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/694136cdd3cd4dd2b0373cc16b7ce862, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/703d607657dd44668c21182a1f58d988, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/be8ec9a19b2042bf95b450e3818ed82a] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp, totalSize=100.8 K 2024-11-27T07:25:14,857 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:14,857 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. files: [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/694136cdd3cd4dd2b0373cc16b7ce862, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/703d607657dd44668c21182a1f58d988, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/be8ec9a19b2042bf95b450e3818ed82a] 2024-11-27T07:25:14,858 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 694136cdd3cd4dd2b0373cc16b7ce862, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1732692310101 2024-11-27T07:25:14,858 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:25:14,858 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1541): 225833644b1bebd5fa53ae861d7c7070/B is initiating minor compaction (all files) 2024-11-27T07:25:14,858 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 225833644b1bebd5fa53ae861d7c7070/B in TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:14,858 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/2013697e01e247988309d7b18630425b, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/c1baa9adb92b4694b53c0598fdf6183d, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/9af2d50204054a1e8d4f62791916fd9a] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp, totalSize=36.8 K 2024-11-27T07:25:14,858 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 703d607657dd44668c21182a1f58d988, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1732692311300 2024-11-27T07:25:14,859 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2013697e01e247988309d7b18630425b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1732692310101 2024-11-27T07:25:14,859 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting be8ec9a19b2042bf95b450e3818ed82a, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1732692312485 2024-11-27T07:25:14,859 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1baa9adb92b4694b53c0598fdf6183d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1732692311300 2024-11-27T07:25:14,860 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9af2d50204054a1e8d4f62791916fd9a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1732692312485 2024-11-27T07:25:14,868 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=225833644b1bebd5fa53ae861d7c7070] 2024-11-27T07:25:14,869 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 225833644b1bebd5fa53ae861d7c7070#B#compaction#177 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:25:14,870 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/620af9d036e542d98858bb0bc151caa9 is 50, key is test_row_0/B:col10/1732692312505/Put/seqid=0 2024-11-27T07:25:14,873 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127308d5bba51674d7ea9e3ae16d2560c5c_225833644b1bebd5fa53ae861d7c7070 store=[table=TestAcidGuarantees family=A region=225833644b1bebd5fa53ae861d7c7070] 2024-11-27T07:25:14,875 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127308d5bba51674d7ea9e3ae16d2560c5c_225833644b1bebd5fa53ae861d7c7070, store=[table=TestAcidGuarantees family=A region=225833644b1bebd5fa53ae861d7c7070] 2024-11-27T07:25:14,875 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127308d5bba51674d7ea9e3ae16d2560c5c_225833644b1bebd5fa53ae861d7c7070 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=225833644b1bebd5fa53ae861d7c7070] 2024-11-27T07:25:14,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742043_1219 (size=13187) 2024-11-27T07:25:14,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742044_1220 (size=4469) 2024-11-27T07:25:14,993 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T07:25:14,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:25:14,993 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 225833644b1bebd5fa53ae861d7c7070 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-27T07:25:15,007 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127c859b2626c6b40c0b178c95b2e822acc_225833644b1bebd5fa53ae861d7c7070 is 50, key is test_row_0/A:col10/1732692314991/Put/seqid=0 2024-11-27T07:25:15,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742045_1221 (size=14994) 2024-11-27T07:25:15,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:15,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36312 deadline: 1732692375012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:15,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:15,020 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:15,020 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:15,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36298 deadline: 1732692375013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:15,020 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:15,020 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:15,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:15,020 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:15,021 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:15,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36314 deadline: 1732692375019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:15,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:15,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:15,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:15,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:15,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36312 deadline: 1732692375129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:15,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:15,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36314 deadline: 1732692375129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:15,131 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:15,131 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:15,131 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:15,131 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:15,131 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:15,131 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:15,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:15,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36298 deadline: 1732692375129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:15,135 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:15,135 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:15,135 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:15,203 INFO [Thread-600 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-600. 2024-11-27T07:25:15,203 DEBUG [Thread-600 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T07:25:15,203 DEBUG [Thread-600 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:25:15,203 DEBUG [Thread-600 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:25:15,203 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T07:25:15,206 INFO [Thread-602 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-602. 2024-11-27T07:25:15,206 DEBUG [Thread-602 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T07:25:15,206 DEBUG [Thread-602 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:25:15,206 DEBUG [Thread-602 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:25:15,206 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T07:25:15,208 INFO [Thread-598 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-598. 2024-11-27T07:25:15,208 DEBUG [Thread-598 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T07:25:15,208 DEBUG [Thread-598 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:25:15,208 DEBUG [Thread-598 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:25:15,208 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T07:25:15,213 INFO [Thread-604 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-604. 2024-11-27T07:25:15,213 DEBUG [Thread-604 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T07:25:15,213 DEBUG [Thread-604 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:25:15,213 DEBUG [Thread-604 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:25:15,213 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T07:25:15,309 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 225833644b1bebd5fa53ae861d7c7070#A#compaction#178 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:25:15,310 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/98015249a9114ca5aa0ecd54a467466f is 175, key is test_row_0/A:col10/1732692312505/Put/seqid=0 2024-11-27T07:25:15,311 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/620af9d036e542d98858bb0bc151caa9 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/620af9d036e542d98858bb0bc151caa9 2024-11-27T07:25:15,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742046_1222 (size=32141) 2024-11-27T07:25:15,320 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 225833644b1bebd5fa53ae861d7c7070/B of 225833644b1bebd5fa53ae861d7c7070 into 620af9d036e542d98858bb0bc151caa9(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:25:15,320 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 225833644b1bebd5fa53ae861d7c7070: 2024-11-27T07:25:15,320 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., storeName=225833644b1bebd5fa53ae861d7c7070/B, priority=13, startTime=1732692314855; duration=0sec 2024-11-27T07:25:15,320 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:25:15,320 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 225833644b1bebd5fa53ae861d7c7070:B 2024-11-27T07:25:15,321 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:25:15,322 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:25:15,322 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1541): 225833644b1bebd5fa53ae861d7c7070/C is initiating minor compaction (all files) 2024-11-27T07:25:15,322 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 225833644b1bebd5fa53ae861d7c7070/C in TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:15,322 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/8ea0713f77d3410e8710bd632ded4037, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/6e09bf5c89c94b709bf5c434610a2858, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/dfa9211a9323494999c9576c608d7074] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp, totalSize=36.8 K 2024-11-27T07:25:15,322 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ea0713f77d3410e8710bd632ded4037, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1732692310101 2024-11-27T07:25:15,323 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e09bf5c89c94b709bf5c434610a2858, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1732692311300 2024-11-27T07:25:15,323 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting dfa9211a9323494999c9576c608d7074, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1732692312485 2024-11-27T07:25:15,330 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 225833644b1bebd5fa53ae861d7c7070#C#compaction#180 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:25:15,330 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/0c78f58e75ff4b228976c7fdf507296f is 50, key is test_row_0/C:col10/1732692312505/Put/seqid=0 2024-11-27T07:25:15,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742047_1223 (size=13187) 2024-11-27T07:25:15,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:15,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:15,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:15,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36314 deadline: 1732692375339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:15,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36312 deadline: 1732692375339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:15,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36298 deadline: 1732692375339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:15,339 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:15,339 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:15,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:15,340 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:15,340 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:15,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:15,340 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:15,340 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:15,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:15,419 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:25:15,423 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127c859b2626c6b40c0b178c95b2e822acc_225833644b1bebd5fa53ae861d7c7070 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127c859b2626c6b40c0b178c95b2e822acc_225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:25:15,424 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/de41599f60104ffa9eaa3b25c1f91512, store: [table=TestAcidGuarantees family=A region=225833644b1bebd5fa53ae861d7c7070] 2024-11-27T07:25:15,424 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/de41599f60104ffa9eaa3b25c1f91512 is 175, key is test_row_0/A:col10/1732692314991/Put/seqid=0 2024-11-27T07:25:15,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742048_1224 (size=39949) 2024-11-27T07:25:15,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-27T07:25:15,429 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T07:25:15,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:15,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36298 deadline: 1732692375650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:15,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:15,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:15,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36312 deadline: 1732692375650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:15,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36314 deadline: 1732692375650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:15,651 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:15,652 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:15,652 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:15,652 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:15,652 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:15,652 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:15,652 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:15,652 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:15,652 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:15,722 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/98015249a9114ca5aa0ecd54a467466f as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/98015249a9114ca5aa0ecd54a467466f 2024-11-27T07:25:15,728 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 225833644b1bebd5fa53ae861d7c7070/A of 225833644b1bebd5fa53ae861d7c7070 into 98015249a9114ca5aa0ecd54a467466f(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:25:15,728 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 225833644b1bebd5fa53ae861d7c7070: 2024-11-27T07:25:15,728 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., storeName=225833644b1bebd5fa53ae861d7c7070/A, priority=13, startTime=1732692314854; duration=0sec 2024-11-27T07:25:15,728 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:25:15,728 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 225833644b1bebd5fa53ae861d7c7070:A 2024-11-27T07:25:15,739 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/0c78f58e75ff4b228976c7fdf507296f as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/0c78f58e75ff4b228976c7fdf507296f 2024-11-27T07:25:15,744 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 225833644b1bebd5fa53ae861d7c7070/C of 225833644b1bebd5fa53ae861d7c7070 into 0c78f58e75ff4b228976c7fdf507296f(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:25:15,744 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 225833644b1bebd5fa53ae861d7c7070: 2024-11-27T07:25:15,744 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., storeName=225833644b1bebd5fa53ae861d7c7070/C, priority=13, startTime=1732692314855; duration=0sec 2024-11-27T07:25:15,744 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:25:15,744 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 225833644b1bebd5fa53ae861d7c7070:C 2024-11-27T07:25:15,830 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=394, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/de41599f60104ffa9eaa3b25c1f91512 2024-11-27T07:25:15,844 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/80e6c984c76f4eefa0c09459f61a0b5f is 50, key is test_row_0/B:col10/1732692314991/Put/seqid=0 2024-11-27T07:25:15,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742049_1225 (size=12301) 2024-11-27T07:25:16,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:16,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:16,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:16,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36312 deadline: 1732692376159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:16,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36298 deadline: 1732692376159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:16,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36314 deadline: 1732692376159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:16,160 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:16,160 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:16,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:16,160 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:16,160 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:16,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:16,160 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:16,160 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:16,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:16,250 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/80e6c984c76f4eefa0c09459f61a0b5f 2024-11-27T07:25:16,260 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/529739014bee4ddf96b3a09fe1effb5b is 50, key is test_row_0/C:col10/1732692314991/Put/seqid=0 2024-11-27T07:25:16,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742050_1226 (size=12301) 2024-11-27T07:25:16,666 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/529739014bee4ddf96b3a09fe1effb5b 2024-11-27T07:25:16,676 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/de41599f60104ffa9eaa3b25c1f91512 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/de41599f60104ffa9eaa3b25c1f91512 2024-11-27T07:25:16,681 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/de41599f60104ffa9eaa3b25c1f91512, entries=200, sequenceid=394, filesize=39.0 K 2024-11-27T07:25:16,682 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/80e6c984c76f4eefa0c09459f61a0b5f as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/80e6c984c76f4eefa0c09459f61a0b5f 2024-11-27T07:25:16,686 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/80e6c984c76f4eefa0c09459f61a0b5f, entries=150, sequenceid=394, filesize=12.0 K 2024-11-27T07:25:16,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/529739014bee4ddf96b3a09fe1effb5b as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/529739014bee4ddf96b3a09fe1effb5b 2024-11-27T07:25:16,689 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/529739014bee4ddf96b3a09fe1effb5b, entries=150, sequenceid=394, filesize=12.0 K 2024-11-27T07:25:16,690 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 225833644b1bebd5fa53ae861d7c7070 in 1697ms, sequenceid=394, compaction requested=false 2024-11-27T07:25:16,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 225833644b1bebd5fa53ae861d7c7070: 2024-11-27T07:25:16,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:25:16,710 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T07:25:16,710 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 225833644b1bebd5fa53ae861d7c7070 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-27T07:25:16,710 INFO [Thread-591 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-591. 2024-11-27T07:25:16,710 DEBUG [Thread-591 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T07:25:16,710 DEBUG [Thread-591 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:25:16,710 DEBUG [Thread-591 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:25:16,710 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T07:25:16,715 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411275010b4ff75224a1980743a11dc033ce6_225833644b1bebd5fa53ae861d7c7070 is 50, key is test_row_0/A:col10/1732692315003/Put/seqid=0 2024-11-27T07:25:16,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742051_1227 (size=12454) 2024-11-27T07:25:17,119 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:25:17,123 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411275010b4ff75224a1980743a11dc033ce6_225833644b1bebd5fa53ae861d7c7070 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411275010b4ff75224a1980743a11dc033ce6_225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:25:17,124 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/7bdcca53b55240f183d626107690ec2f, store: [table=TestAcidGuarantees family=A region=225833644b1bebd5fa53ae861d7c7070] 2024-11-27T07:25:17,125 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/7bdcca53b55240f183d626107690ec2f is 175, key is test_row_0/A:col10/1732692315003/Put/seqid=0 2024-11-27T07:25:17,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742052_1228 (size=31255) 2024-11-27T07:25:17,170 INFO [Thread-587 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-587. 2024-11-27T07:25:17,170 DEBUG [Thread-587 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T07:25:17,170 DEBUG [Thread-587 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:25:17,170 DEBUG [Thread-587 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:25:17,170 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T07:25:17,181 INFO [Thread-589 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-589. 2024-11-27T07:25:17,181 INFO [Thread-593 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-593. 2024-11-27T07:25:17,181 DEBUG [Thread-589 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T07:25:17,181 DEBUG [Thread-593 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T07:25:17,181 DEBUG [Thread-589 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:25:17,181 DEBUG [Thread-593 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:25:17,181 DEBUG [Thread-589 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:25:17,181 DEBUG [Thread-593 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:25:17,181 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T07:25:17,181 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T07:25:17,529 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=413, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/7bdcca53b55240f183d626107690ec2f 2024-11-27T07:25:17,536 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/d80ee88b338948edaadf8641ed452c44 is 50, key is test_row_0/B:col10/1732692315003/Put/seqid=0 2024-11-27T07:25:17,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742053_1229 (size=12301) 2024-11-27T07:25:17,941 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/d80ee88b338948edaadf8641ed452c44 2024-11-27T07:25:17,947 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/4923c3df1cc04ae0b241315624bca3fc is 50, key is test_row_0/C:col10/1732692315003/Put/seqid=0 2024-11-27T07:25:17,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742054_1230 (size=12301) 2024-11-27T07:25:17,970 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-27T07:25:18,376 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/4923c3df1cc04ae0b241315624bca3fc 2024-11-27T07:25:18,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/7bdcca53b55240f183d626107690ec2f as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/7bdcca53b55240f183d626107690ec2f 2024-11-27T07:25:18,392 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/7bdcca53b55240f183d626107690ec2f, entries=150, sequenceid=413, filesize=30.5 K 2024-11-27T07:25:18,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/d80ee88b338948edaadf8641ed452c44 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/d80ee88b338948edaadf8641ed452c44 2024-11-27T07:25:18,397 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/d80ee88b338948edaadf8641ed452c44, entries=150, sequenceid=413, filesize=12.0 K 2024-11-27T07:25:18,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/4923c3df1cc04ae0b241315624bca3fc as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/4923c3df1cc04ae0b241315624bca3fc 2024-11-27T07:25:18,401 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/4923c3df1cc04ae0b241315624bca3fc, entries=150, sequenceid=413, filesize=12.0 K 2024-11-27T07:25:18,402 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=20.13 KB/20610 for 225833644b1bebd5fa53ae861d7c7070 in 1692ms, sequenceid=413, compaction requested=true 2024-11-27T07:25:18,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 225833644b1bebd5fa53ae861d7c7070: 2024-11-27T07:25:18,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 225833644b1bebd5fa53ae861d7c7070:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T07:25:18,402 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:25:18,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:25:18,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 225833644b1bebd5fa53ae861d7c7070:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T07:25:18,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:25:18,403 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:25:18,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 225833644b1bebd5fa53ae861d7c7070:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T07:25:18,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:25:18,403 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:25:18,403 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103345 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:25:18,404 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 225833644b1bebd5fa53ae861d7c7070/B is initiating minor compaction (all files) 2024-11-27T07:25:18,404 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1541): 225833644b1bebd5fa53ae861d7c7070/A is initiating minor compaction (all files) 2024-11-27T07:25:18,404 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 225833644b1bebd5fa53ae861d7c7070/A in TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:18,404 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 225833644b1bebd5fa53ae861d7c7070/B in TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:18,404 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/98015249a9114ca5aa0ecd54a467466f, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/de41599f60104ffa9eaa3b25c1f91512, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/7bdcca53b55240f183d626107690ec2f] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp, totalSize=100.9 K 2024-11-27T07:25:18,404 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/620af9d036e542d98858bb0bc151caa9, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/80e6c984c76f4eefa0c09459f61a0b5f, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/d80ee88b338948edaadf8641ed452c44] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp, totalSize=36.9 K 2024-11-27T07:25:18,404 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:18,404 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. files: [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/98015249a9114ca5aa0ecd54a467466f, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/de41599f60104ffa9eaa3b25c1f91512, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/7bdcca53b55240f183d626107690ec2f] 2024-11-27T07:25:18,404 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 620af9d036e542d98858bb0bc151caa9, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1732692312485 2024-11-27T07:25:18,404 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98015249a9114ca5aa0ecd54a467466f, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1732692312485 2024-11-27T07:25:18,404 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 80e6c984c76f4eefa0c09459f61a0b5f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1732692314991 2024-11-27T07:25:18,404 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting de41599f60104ffa9eaa3b25c1f91512, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1732692314356 2024-11-27T07:25:18,405 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting d80ee88b338948edaadf8641ed452c44, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1732692315003 2024-11-27T07:25:18,405 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7bdcca53b55240f183d626107690ec2f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1732692315003 2024-11-27T07:25:18,414 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=225833644b1bebd5fa53ae861d7c7070] 2024-11-27T07:25:18,416 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112766e9e1bab5004cddb5085c355dd07911_225833644b1bebd5fa53ae861d7c7070 store=[table=TestAcidGuarantees family=A region=225833644b1bebd5fa53ae861d7c7070] 2024-11-27T07:25:18,420 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 225833644b1bebd5fa53ae861d7c7070#B#compaction#186 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:25:18,421 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/0622f3049a6e409c9a696226d25e9d14 is 50, key is test_row_0/B:col10/1732692315003/Put/seqid=0 2024-11-27T07:25:18,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742055_1231 (size=13289) 2024-11-27T07:25:18,435 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/0622f3049a6e409c9a696226d25e9d14 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/0622f3049a6e409c9a696226d25e9d14 2024-11-27T07:25:18,440 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 225833644b1bebd5fa53ae861d7c7070/B of 225833644b1bebd5fa53ae861d7c7070 into 0622f3049a6e409c9a696226d25e9d14(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:25:18,440 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 225833644b1bebd5fa53ae861d7c7070: 2024-11-27T07:25:18,440 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., storeName=225833644b1bebd5fa53ae861d7c7070/B, priority=13, startTime=1732692318402; duration=0sec 2024-11-27T07:25:18,440 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:25:18,440 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 225833644b1bebd5fa53ae861d7c7070:B 2024-11-27T07:25:18,440 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:25:18,441 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:25:18,441 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 225833644b1bebd5fa53ae861d7c7070/C is initiating minor compaction (all files) 2024-11-27T07:25:18,442 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 225833644b1bebd5fa53ae861d7c7070/C in TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:18,442 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/0c78f58e75ff4b228976c7fdf507296f, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/529739014bee4ddf96b3a09fe1effb5b, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/4923c3df1cc04ae0b241315624bca3fc] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp, totalSize=36.9 K 2024-11-27T07:25:18,442 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c78f58e75ff4b228976c7fdf507296f, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1732692312485 2024-11-27T07:25:18,442 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 529739014bee4ddf96b3a09fe1effb5b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1732692314991 2024-11-27T07:25:18,442 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 4923c3df1cc04ae0b241315624bca3fc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1732692315003 2024-11-27T07:25:18,449 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112766e9e1bab5004cddb5085c355dd07911_225833644b1bebd5fa53ae861d7c7070, store=[table=TestAcidGuarantees family=A region=225833644b1bebd5fa53ae861d7c7070] 2024-11-27T07:25:18,449 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112766e9e1bab5004cddb5085c355dd07911_225833644b1bebd5fa53ae861d7c7070 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=225833644b1bebd5fa53ae861d7c7070] 2024-11-27T07:25:18,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742056_1232 (size=4469) 2024-11-27T07:25:18,461 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 225833644b1bebd5fa53ae861d7c7070#C#compaction#188 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:25:18,461 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/a90209be2e9748d9a433c1182fb1c83d is 50, key is test_row_0/C:col10/1732692315003/Put/seqid=0 2024-11-27T07:25:18,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742057_1233 (size=13289) 2024-11-27T07:25:18,858 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 225833644b1bebd5fa53ae861d7c7070#A#compaction#187 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:25:18,860 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/c985965fb8dd4c90b149f9e8e0cbe45a is 175, key is test_row_0/A:col10/1732692315003/Put/seqid=0 2024-11-27T07:25:18,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742058_1234 (size=32243) 2024-11-27T07:25:18,871 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/a90209be2e9748d9a433c1182fb1c83d as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/a90209be2e9748d9a433c1182fb1c83d 2024-11-27T07:25:18,877 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 225833644b1bebd5fa53ae861d7c7070/C of 225833644b1bebd5fa53ae861d7c7070 into a90209be2e9748d9a433c1182fb1c83d(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:25:18,877 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 225833644b1bebd5fa53ae861d7c7070: 2024-11-27T07:25:18,877 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., storeName=225833644b1bebd5fa53ae861d7c7070/C, priority=13, startTime=1732692318403; duration=0sec 2024-11-27T07:25:18,877 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:25:18,877 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 225833644b1bebd5fa53ae861d7c7070:C 2024-11-27T07:25:19,281 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/c985965fb8dd4c90b149f9e8e0cbe45a as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/c985965fb8dd4c90b149f9e8e0cbe45a 2024-11-27T07:25:19,287 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 225833644b1bebd5fa53ae861d7c7070/A of 225833644b1bebd5fa53ae861d7c7070 into c985965fb8dd4c90b149f9e8e0cbe45a(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:25:19,287 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 225833644b1bebd5fa53ae861d7c7070: 2024-11-27T07:25:19,288 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., storeName=225833644b1bebd5fa53ae861d7c7070/A, priority=13, startTime=1732692318402; duration=0sec 2024-11-27T07:25:19,288 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:25:19,288 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 225833644b1bebd5fa53ae861d7c7070:A 2024-11-27T07:25:24,400 INFO [Thread-595 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-595. 2024-11-27T07:25:24,400 DEBUG [Thread-595 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T07:25:24,400 DEBUG [Thread-595 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:25:24,401 DEBUG [Thread-595 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:25:24,401 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T07:25:24,401 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-27T07:25:24,401 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 47 2024-11-27T07:25:24,401 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2024-11-27T07:25:24,401 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 89 2024-11-27T07:25:24,401 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-11-27T07:25:24,401 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 89 2024-11-27T07:25:24,401 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-27T07:25:24,401 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3699 2024-11-27T07:25:24,401 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3634 2024-11-27T07:25:24,401 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-27T07:25:24,401 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1632 2024-11-27T07:25:24,401 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4893 rows 2024-11-27T07:25:24,401 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1629 2024-11-27T07:25:24,401 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4887 rows 2024-11-27T07:25:24,401 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T07:25:24,401 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T07:25:24,401 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:25:24,401 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:25:24,401 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T07:25:24,402 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-27T07:25:24,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-27T07:25:24,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-27T07:25:24,406 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732692324406"}]},"ts":"1732692324406"} 2024-11-27T07:25:24,408 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-27T07:25:24,408 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-27T07:25:24,409 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-27T07:25:24,411 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=225833644b1bebd5fa53ae861d7c7070, UNASSIGN}] 2024-11-27T07:25:24,412 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=225833644b1bebd5fa53ae861d7c7070, UNASSIGN 2024-11-27T07:25:24,413 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=225833644b1bebd5fa53ae861d7c7070, regionState=CLOSING, regionLocation=289954c9202d,42117,1732692259671 2024-11-27T07:25:24,415 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=225833644b1bebd5fa53ae861d7c7070, UNASSIGN because future has completed 2024-11-27T07:25:24,415 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T07:25:24,415 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; CloseRegionProcedure 225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671}] 2024-11-27T07:25:24,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-27T07:25:24,568 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(122): Close 225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:25:24,568 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T07:25:24,568 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1722): Closing 225833644b1bebd5fa53ae861d7c7070, disabling compactions & flushes 2024-11-27T07:25:24,568 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:24,568 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:24,568 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. after waiting 0 ms 2024-11-27T07:25:24,568 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:24,569 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(2902): Flushing 225833644b1bebd5fa53ae861d7c7070 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-27T07:25:24,576 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411277cb2443f69fe436c8520d37d11029dde_225833644b1bebd5fa53ae861d7c7070 is 50, key is test_row_0/A:col10/1732692317169/Put/seqid=0 2024-11-27T07:25:24,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742059_1235 (size=12454) 2024-11-27T07:25:24,581 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:25:24,586 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411277cb2443f69fe436c8520d37d11029dde_225833644b1bebd5fa53ae861d7c7070 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411277cb2443f69fe436c8520d37d11029dde_225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:25:24,587 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/a79e59aec0af4b8c9c1fcd307de28e27, store: [table=TestAcidGuarantees family=A region=225833644b1bebd5fa53ae861d7c7070] 2024-11-27T07:25:24,588 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/a79e59aec0af4b8c9c1fcd307de28e27 is 175, key is test_row_0/A:col10/1732692317169/Put/seqid=0 2024-11-27T07:25:24,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742060_1236 (size=31255) 2024-11-27T07:25:24,597 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=423, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/a79e59aec0af4b8c9c1fcd307de28e27 2024-11-27T07:25:24,606 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/ccecf7407b6c485983fd6ab15986d684 is 50, key is test_row_0/B:col10/1732692317169/Put/seqid=0 2024-11-27T07:25:24,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742061_1237 (size=12301) 2024-11-27T07:25:24,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-27T07:25:25,015 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/ccecf7407b6c485983fd6ab15986d684 2024-11-27T07:25:25,026 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/bc35257bd5884e748285fbec85dfe306 is 50, key is test_row_0/C:col10/1732692317169/Put/seqid=0 2024-11-27T07:25:25,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-27T07:25:25,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742062_1238 (size=12301) 2024-11-27T07:25:25,431 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/bc35257bd5884e748285fbec85dfe306 2024-11-27T07:25:25,435 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/A/a79e59aec0af4b8c9c1fcd307de28e27 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/a79e59aec0af4b8c9c1fcd307de28e27 2024-11-27T07:25:25,439 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/a79e59aec0af4b8c9c1fcd307de28e27, entries=150, sequenceid=423, filesize=30.5 K 2024-11-27T07:25:25,440 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/B/ccecf7407b6c485983fd6ab15986d684 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/ccecf7407b6c485983fd6ab15986d684 2024-11-27T07:25:25,444 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/ccecf7407b6c485983fd6ab15986d684, entries=150, sequenceid=423, filesize=12.0 K 2024-11-27T07:25:25,445 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/.tmp/C/bc35257bd5884e748285fbec85dfe306 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/bc35257bd5884e748285fbec85dfe306 2024-11-27T07:25:25,449 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/bc35257bd5884e748285fbec85dfe306, entries=150, sequenceid=423, filesize=12.0 K 2024-11-27T07:25:25,450 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 225833644b1bebd5fa53ae861d7c7070 in 882ms, sequenceid=423, compaction requested=false 2024-11-27T07:25:25,450 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/f2f55e9ff30a4beaa03dd8e1018f4660, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/2e718dcb21254a48af41e0ae08b92261, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/29ac21a055c0427f88779f2b67b83ddb, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/6c90e21305ac40da83d99fa62732443b, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/f965528058614b93999d39c30bff5895, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/7c61ad07bbbe4f3683ef9d7658394644, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/940f4c44b1c044dfbeb89c0c6988c8e8, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/1b941f695de147639c5ea5ec69814fb3, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/9d8168d9b6ab4d5e93fee0bbaaee8dd3, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/f3ad16b9b08c42baa75299574525b972, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/6e2ab6dd0050461e9b329d880f47f866, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/9703987f37e044c3899aaad633b1da47, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/88b98d16c4c242bcb6433521083bd2af, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/dd8bba1c5e934853a2d2e53931ea71df, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/04ea3527339c43a0811cabee284c1a09, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/0fe3d1c4379f4149a21e052bffdf3c67, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/553770897dfa44efaa9376f7cd79b6f7, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/44d0dabb8cc64cd6801a6062ad26dbca, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/8afef07fb8734e938c86b4614afa0fec, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/2061d197b8d8410f9f44a727cdf142e8, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/41bcc16a36db4e8f8895fc9ed27334d9, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/9e301b2d30ed4a3ca98b2e7cf102b186, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/bcc6d9140ed841be83f14ffe19cd9484, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/694136cdd3cd4dd2b0373cc16b7ce862, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/703d607657dd44668c21182a1f58d988, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/be8ec9a19b2042bf95b450e3818ed82a, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/98015249a9114ca5aa0ecd54a467466f, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/de41599f60104ffa9eaa3b25c1f91512, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/7bdcca53b55240f183d626107690ec2f] to archive 2024-11-27T07:25:25,451 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T07:25:25,453 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/f2f55e9ff30a4beaa03dd8e1018f4660 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/f2f55e9ff30a4beaa03dd8e1018f4660 2024-11-27T07:25:25,454 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/2e718dcb21254a48af41e0ae08b92261 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/2e718dcb21254a48af41e0ae08b92261 2024-11-27T07:25:25,455 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/29ac21a055c0427f88779f2b67b83ddb to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/29ac21a055c0427f88779f2b67b83ddb 2024-11-27T07:25:25,456 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/6c90e21305ac40da83d99fa62732443b to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/6c90e21305ac40da83d99fa62732443b 2024-11-27T07:25:25,457 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/f965528058614b93999d39c30bff5895 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/f965528058614b93999d39c30bff5895 2024-11-27T07:25:25,459 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/7c61ad07bbbe4f3683ef9d7658394644 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/7c61ad07bbbe4f3683ef9d7658394644 2024-11-27T07:25:25,460 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/940f4c44b1c044dfbeb89c0c6988c8e8 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/940f4c44b1c044dfbeb89c0c6988c8e8 2024-11-27T07:25:25,461 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/1b941f695de147639c5ea5ec69814fb3 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/1b941f695de147639c5ea5ec69814fb3 2024-11-27T07:25:25,462 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/9d8168d9b6ab4d5e93fee0bbaaee8dd3 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/9d8168d9b6ab4d5e93fee0bbaaee8dd3 2024-11-27T07:25:25,463 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/f3ad16b9b08c42baa75299574525b972 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/f3ad16b9b08c42baa75299574525b972 2024-11-27T07:25:25,464 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/6e2ab6dd0050461e9b329d880f47f866 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/6e2ab6dd0050461e9b329d880f47f866 2024-11-27T07:25:25,465 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/9703987f37e044c3899aaad633b1da47 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/9703987f37e044c3899aaad633b1da47 2024-11-27T07:25:25,466 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/88b98d16c4c242bcb6433521083bd2af to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/88b98d16c4c242bcb6433521083bd2af 2024-11-27T07:25:25,467 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/dd8bba1c5e934853a2d2e53931ea71df to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/dd8bba1c5e934853a2d2e53931ea71df 2024-11-27T07:25:25,469 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/04ea3527339c43a0811cabee284c1a09 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/04ea3527339c43a0811cabee284c1a09 2024-11-27T07:25:25,470 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/0fe3d1c4379f4149a21e052bffdf3c67 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/0fe3d1c4379f4149a21e052bffdf3c67 2024-11-27T07:25:25,471 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/553770897dfa44efaa9376f7cd79b6f7 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/553770897dfa44efaa9376f7cd79b6f7 2024-11-27T07:25:25,472 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/44d0dabb8cc64cd6801a6062ad26dbca to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/44d0dabb8cc64cd6801a6062ad26dbca 2024-11-27T07:25:25,473 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/8afef07fb8734e938c86b4614afa0fec to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/8afef07fb8734e938c86b4614afa0fec 2024-11-27T07:25:25,474 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/2061d197b8d8410f9f44a727cdf142e8 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/2061d197b8d8410f9f44a727cdf142e8 2024-11-27T07:25:25,475 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/41bcc16a36db4e8f8895fc9ed27334d9 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/41bcc16a36db4e8f8895fc9ed27334d9 2024-11-27T07:25:25,476 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/9e301b2d30ed4a3ca98b2e7cf102b186 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/9e301b2d30ed4a3ca98b2e7cf102b186 2024-11-27T07:25:25,477 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/bcc6d9140ed841be83f14ffe19cd9484 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/bcc6d9140ed841be83f14ffe19cd9484 2024-11-27T07:25:25,478 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/694136cdd3cd4dd2b0373cc16b7ce862 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/694136cdd3cd4dd2b0373cc16b7ce862 2024-11-27T07:25:25,479 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/703d607657dd44668c21182a1f58d988 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/703d607657dd44668c21182a1f58d988 2024-11-27T07:25:25,480 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/be8ec9a19b2042bf95b450e3818ed82a to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/be8ec9a19b2042bf95b450e3818ed82a 2024-11-27T07:25:25,481 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/98015249a9114ca5aa0ecd54a467466f to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/98015249a9114ca5aa0ecd54a467466f 2024-11-27T07:25:25,482 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/de41599f60104ffa9eaa3b25c1f91512 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/de41599f60104ffa9eaa3b25c1f91512 2024-11-27T07:25:25,483 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/7bdcca53b55240f183d626107690ec2f to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/7bdcca53b55240f183d626107690ec2f 2024-11-27T07:25:25,484 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/30b91302f41b4211b9d13219d2384e57, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/09c2adf80b6c483a97d5d5c288a2b975, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/d7c999b8003748c3b983cb52d30400ee, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/f5fe59552cb748e983308e4283c0e01f, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/1a9802761b8c4a8caef051adff35c284, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/79e6b404310b4e14ac394562c8b3b2e3, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/5ce9338c195541b6aa715529e4696e08, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/fb0eb8e459c74b7a9944ad9f1098deea, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/4e6df53335d34f58ae16edddf9061526, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/3468c3fd380f461ba5417f79aa1393b7, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/c57c343b78954d3e9e64e79f44800226, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/613cfd71f60741bdb8f44c3e28e6138d, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/4543c195626c4d35b19ce170276461d1, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/b57b193b60ed416f9f021037a751afd1, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/c1a936766ced4e3dae4b0457ad1261e3, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/304326091b0d4870973e679e31ec4a81, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/a9371fd13d2844148a1c98a03e12175c, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/04fd0fd79e844c938ed7eb8b75582ad0, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/b7ae8c95158b433796629687eaeb6b04, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/0b21f98c58b84dd8bdc3211156783450, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/9aaa23cfdbfd4fffbb58a175c305adf6, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/067cfbc8e62d4ca4bbf8c276b5b83153, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/2013697e01e247988309d7b18630425b, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/34d33d3d04d24b3ab243ed8ded171342, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/c1baa9adb92b4694b53c0598fdf6183d, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/620af9d036e542d98858bb0bc151caa9, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/9af2d50204054a1e8d4f62791916fd9a, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/80e6c984c76f4eefa0c09459f61a0b5f, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/d80ee88b338948edaadf8641ed452c44] to archive 2024-11-27T07:25:25,485 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T07:25:25,487 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/30b91302f41b4211b9d13219d2384e57 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/30b91302f41b4211b9d13219d2384e57 2024-11-27T07:25:25,488 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/09c2adf80b6c483a97d5d5c288a2b975 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/09c2adf80b6c483a97d5d5c288a2b975 2024-11-27T07:25:25,489 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/d7c999b8003748c3b983cb52d30400ee to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/d7c999b8003748c3b983cb52d30400ee 2024-11-27T07:25:25,490 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/f5fe59552cb748e983308e4283c0e01f to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/f5fe59552cb748e983308e4283c0e01f 2024-11-27T07:25:25,491 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/1a9802761b8c4a8caef051adff35c284 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/1a9802761b8c4a8caef051adff35c284 2024-11-27T07:25:25,492 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/79e6b404310b4e14ac394562c8b3b2e3 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/79e6b404310b4e14ac394562c8b3b2e3 2024-11-27T07:25:25,493 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/5ce9338c195541b6aa715529e4696e08 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/5ce9338c195541b6aa715529e4696e08 2024-11-27T07:25:25,494 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/fb0eb8e459c74b7a9944ad9f1098deea to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/fb0eb8e459c74b7a9944ad9f1098deea 2024-11-27T07:25:25,495 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/4e6df53335d34f58ae16edddf9061526 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/4e6df53335d34f58ae16edddf9061526 2024-11-27T07:25:25,496 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/3468c3fd380f461ba5417f79aa1393b7 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/3468c3fd380f461ba5417f79aa1393b7 2024-11-27T07:25:25,497 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/c57c343b78954d3e9e64e79f44800226 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/c57c343b78954d3e9e64e79f44800226 2024-11-27T07:25:25,497 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/613cfd71f60741bdb8f44c3e28e6138d to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/613cfd71f60741bdb8f44c3e28e6138d 2024-11-27T07:25:25,498 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/4543c195626c4d35b19ce170276461d1 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/4543c195626c4d35b19ce170276461d1 2024-11-27T07:25:25,499 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/b57b193b60ed416f9f021037a751afd1 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/b57b193b60ed416f9f021037a751afd1 2024-11-27T07:25:25,501 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/c1a936766ced4e3dae4b0457ad1261e3 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/c1a936766ced4e3dae4b0457ad1261e3 2024-11-27T07:25:25,502 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/304326091b0d4870973e679e31ec4a81 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/304326091b0d4870973e679e31ec4a81 2024-11-27T07:25:25,503 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/a9371fd13d2844148a1c98a03e12175c to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/a9371fd13d2844148a1c98a03e12175c 2024-11-27T07:25:25,504 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/04fd0fd79e844c938ed7eb8b75582ad0 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/04fd0fd79e844c938ed7eb8b75582ad0 2024-11-27T07:25:25,505 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/b7ae8c95158b433796629687eaeb6b04 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/b7ae8c95158b433796629687eaeb6b04 2024-11-27T07:25:25,506 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/0b21f98c58b84dd8bdc3211156783450 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/0b21f98c58b84dd8bdc3211156783450 2024-11-27T07:25:25,507 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/9aaa23cfdbfd4fffbb58a175c305adf6 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/9aaa23cfdbfd4fffbb58a175c305adf6 2024-11-27T07:25:25,508 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/067cfbc8e62d4ca4bbf8c276b5b83153 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/067cfbc8e62d4ca4bbf8c276b5b83153 2024-11-27T07:25:25,509 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/2013697e01e247988309d7b18630425b to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/2013697e01e247988309d7b18630425b 2024-11-27T07:25:25,510 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/34d33d3d04d24b3ab243ed8ded171342 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/34d33d3d04d24b3ab243ed8ded171342 2024-11-27T07:25:25,511 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/c1baa9adb92b4694b53c0598fdf6183d to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/c1baa9adb92b4694b53c0598fdf6183d 2024-11-27T07:25:25,513 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/620af9d036e542d98858bb0bc151caa9 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/620af9d036e542d98858bb0bc151caa9 2024-11-27T07:25:25,514 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/9af2d50204054a1e8d4f62791916fd9a to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/9af2d50204054a1e8d4f62791916fd9a 2024-11-27T07:25:25,515 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/80e6c984c76f4eefa0c09459f61a0b5f to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/80e6c984c76f4eefa0c09459f61a0b5f 2024-11-27T07:25:25,516 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/d80ee88b338948edaadf8641ed452c44 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/d80ee88b338948edaadf8641ed452c44 2024-11-27T07:25:25,517 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/50073d7d11ea45c4bdb1951d8c0608b4, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/96c1c4adf48e454780131cebf161acbf, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/3795192dd17d42aeba63ad4baf94ec79, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/2276c55338654b50a52106db051a31e7, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/35bd2437cc9e48909ac8767158bb895e, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/8754fa27f2fe4423a45531f7849a375e, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/c3eec9f4933a4b29a4874ff2c96443a3, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/ef60204e57024d428fb4e501961dad51, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/51fb5a4288d04cfcb61cf14210d7b1ca, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/a7b7f21cb1e846ebb8171444ce12c453, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/42cce269fb91498abaedfffd0a9b0fc2, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/8b63ee95925f4d7d8538ede3124c0363, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/01330647c4d542798dcb9938ee6beac2, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/43b4dc4492244102ac41bc13d5d2642f, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/9e203d913fac497297d452a0c3efaef9, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/3c3aec40063e413ca89899d65cf216c4, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/d816250e37154816a59d15432331e197, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/c24a3590804a4e26a9ac69becac9c188, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/f4571adb23b84ef68edfc6a17a32d0f4, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/f7f6b4ed072c407c9ebb8cbe51de8525, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/fd0ee664bedd4551b9704c615c1efd9c, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/ae7260d3a3bc4d55ad3a81ad26e3790a, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/8ea0713f77d3410e8710bd632ded4037, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/b4402022ac624c97a3a1576118541ebd, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/6e09bf5c89c94b709bf5c434610a2858, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/0c78f58e75ff4b228976c7fdf507296f, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/dfa9211a9323494999c9576c608d7074, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/529739014bee4ddf96b3a09fe1effb5b, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/4923c3df1cc04ae0b241315624bca3fc] to archive 2024-11-27T07:25:25,518 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T07:25:25,519 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/50073d7d11ea45c4bdb1951d8c0608b4 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/50073d7d11ea45c4bdb1951d8c0608b4 2024-11-27T07:25:25,520 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/96c1c4adf48e454780131cebf161acbf to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/96c1c4adf48e454780131cebf161acbf 2024-11-27T07:25:25,521 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/3795192dd17d42aeba63ad4baf94ec79 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/3795192dd17d42aeba63ad4baf94ec79 2024-11-27T07:25:25,522 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/2276c55338654b50a52106db051a31e7 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/2276c55338654b50a52106db051a31e7 2024-11-27T07:25:25,522 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/35bd2437cc9e48909ac8767158bb895e to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/35bd2437cc9e48909ac8767158bb895e 2024-11-27T07:25:25,523 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/8754fa27f2fe4423a45531f7849a375e to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/8754fa27f2fe4423a45531f7849a375e 2024-11-27T07:25:25,524 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/c3eec9f4933a4b29a4874ff2c96443a3 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/c3eec9f4933a4b29a4874ff2c96443a3 2024-11-27T07:25:25,525 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/ef60204e57024d428fb4e501961dad51 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/ef60204e57024d428fb4e501961dad51 2024-11-27T07:25:25,527 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/51fb5a4288d04cfcb61cf14210d7b1ca to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/51fb5a4288d04cfcb61cf14210d7b1ca 2024-11-27T07:25:25,528 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/a7b7f21cb1e846ebb8171444ce12c453 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/a7b7f21cb1e846ebb8171444ce12c453 2024-11-27T07:25:25,529 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/42cce269fb91498abaedfffd0a9b0fc2 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/42cce269fb91498abaedfffd0a9b0fc2 2024-11-27T07:25:25,530 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/8b63ee95925f4d7d8538ede3124c0363 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/8b63ee95925f4d7d8538ede3124c0363 2024-11-27T07:25:25,531 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/01330647c4d542798dcb9938ee6beac2 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/01330647c4d542798dcb9938ee6beac2 2024-11-27T07:25:25,533 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/43b4dc4492244102ac41bc13d5d2642f to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/43b4dc4492244102ac41bc13d5d2642f 2024-11-27T07:25:25,534 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/9e203d913fac497297d452a0c3efaef9 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/9e203d913fac497297d452a0c3efaef9 2024-11-27T07:25:25,535 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/3c3aec40063e413ca89899d65cf216c4 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/3c3aec40063e413ca89899d65cf216c4 2024-11-27T07:25:25,536 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/d816250e37154816a59d15432331e197 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/d816250e37154816a59d15432331e197 2024-11-27T07:25:25,537 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/c24a3590804a4e26a9ac69becac9c188 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/c24a3590804a4e26a9ac69becac9c188 2024-11-27T07:25:25,538 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/f4571adb23b84ef68edfc6a17a32d0f4 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/f4571adb23b84ef68edfc6a17a32d0f4 2024-11-27T07:25:25,539 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/f7f6b4ed072c407c9ebb8cbe51de8525 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/f7f6b4ed072c407c9ebb8cbe51de8525 2024-11-27T07:25:25,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-27T07:25:25,540 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/fd0ee664bedd4551b9704c615c1efd9c to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/fd0ee664bedd4551b9704c615c1efd9c 2024-11-27T07:25:25,541 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/ae7260d3a3bc4d55ad3a81ad26e3790a to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/ae7260d3a3bc4d55ad3a81ad26e3790a 2024-11-27T07:25:25,542 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/8ea0713f77d3410e8710bd632ded4037 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/8ea0713f77d3410e8710bd632ded4037 2024-11-27T07:25:25,543 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/b4402022ac624c97a3a1576118541ebd to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/b4402022ac624c97a3a1576118541ebd 2024-11-27T07:25:25,544 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/6e09bf5c89c94b709bf5c434610a2858 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/6e09bf5c89c94b709bf5c434610a2858 2024-11-27T07:25:25,545 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/0c78f58e75ff4b228976c7fdf507296f to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/0c78f58e75ff4b228976c7fdf507296f 2024-11-27T07:25:25,546 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/dfa9211a9323494999c9576c608d7074 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/dfa9211a9323494999c9576c608d7074 2024-11-27T07:25:25,546 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/529739014bee4ddf96b3a09fe1effb5b to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/529739014bee4ddf96b3a09fe1effb5b 2024-11-27T07:25:25,547 DEBUG [StoreCloser-TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/4923c3df1cc04ae0b241315624bca3fc to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/4923c3df1cc04ae0b241315624bca3fc 2024-11-27T07:25:25,551 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/recovered.edits/426.seqid, newMaxSeqId=426, maxSeqId=4 2024-11-27T07:25:25,552 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. 2024-11-27T07:25:25,552 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1676): Region close journal for 225833644b1bebd5fa53ae861d7c7070: Waiting for close lock at 1732692324568Running coprocessor pre-close hooks at 1732692324568Disabling compacts and flushes for region at 1732692324568Disabling writes for close at 1732692324568Obtaining lock to block concurrent updates at 1732692324569 (+1 ms)Preparing flush snapshotting stores in 225833644b1bebd5fa53ae861d7c7070 at 1732692324569Finished memstore snapshotting TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1732692324569Flushing stores of TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070. at 1732692324569Flushing 225833644b1bebd5fa53ae861d7c7070/A: creating writer at 1732692324569Flushing mob file 225833644b1bebd5fa53ae861d7c7070/A: appending metadata at 1732692324576 (+7 ms)Flushing mob file 225833644b1bebd5fa53ae861d7c7070/A: closing flushed file at 1732692324576Flushing 225833644b1bebd5fa53ae861d7c7070/A: appending metadata at 1732692324587 (+11 ms)Flushing 225833644b1bebd5fa53ae861d7c7070/A: closing flushed file at 1732692324587Flushing 225833644b1bebd5fa53ae861d7c7070/B: creating writer at 1732692324602 (+15 ms)Flushing 225833644b1bebd5fa53ae861d7c7070/B: appending metadata at 1732692324606 (+4 ms)Flushing 225833644b1bebd5fa53ae861d7c7070/B: closing flushed file at 1732692324606Flushing 225833644b1bebd5fa53ae861d7c7070/C: creating writer at 1732692325022 (+416 ms)Flushing 225833644b1bebd5fa53ae861d7c7070/C: appending metadata at 1732692325025 (+3 ms)Flushing 225833644b1bebd5fa53ae861d7c7070/C: closing flushed file at 1732692325026 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@c131c81: reopening flushed file at 1732692325435 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@fcb4bf3: reopening flushed file at 1732692325440 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7e375b10: reopening flushed file at 1732692325445 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 225833644b1bebd5fa53ae861d7c7070 in 882ms, sequenceid=423, compaction requested=false at 1732692325450 (+5 ms)Writing region close event to WAL at 1732692325549 (+99 ms)Running coprocessor post-close hooks at 1732692325552 (+3 ms)Closed at 1732692325552 2024-11-27T07:25:25,554 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(157): Closed 225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:25:25,554 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=225833644b1bebd5fa53ae861d7c7070, regionState=CLOSED 2024-11-27T07:25:25,555 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; CloseRegionProcedure 225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 because future has completed 2024-11-27T07:25:25,557 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-27T07:25:25,557 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; CloseRegionProcedure 225833644b1bebd5fa53ae861d7c7070, server=289954c9202d,42117,1732692259671 in 1.1410 sec 2024-11-27T07:25:25,559 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-11-27T07:25:25,559 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=225833644b1bebd5fa53ae861d7c7070, UNASSIGN in 1.1470 sec 2024-11-27T07:25:25,560 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-27T07:25:25,561 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.1500 sec 2024-11-27T07:25:25,562 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732692325561"}]},"ts":"1732692325561"} 2024-11-27T07:25:25,563 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-27T07:25:25,563 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-27T07:25:25,564 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.1610 sec 2024-11-27T07:25:26,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-27T07:25:26,560 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-27T07:25:26,562 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-27T07:25:26,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T07:25:26,565 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=62, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T07:25:26,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-27T07:25:26,566 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=62, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T07:25:26,570 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:25:26,575 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A, FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B, FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C, FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/recovered.edits] 2024-11-27T07:25:26,580 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/a79e59aec0af4b8c9c1fcd307de28e27 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/a79e59aec0af4b8c9c1fcd307de28e27 2024-11-27T07:25:26,582 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/c985965fb8dd4c90b149f9e8e0cbe45a to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/A/c985965fb8dd4c90b149f9e8e0cbe45a 2024-11-27T07:25:26,586 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/0622f3049a6e409c9a696226d25e9d14 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/0622f3049a6e409c9a696226d25e9d14 2024-11-27T07:25:26,589 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/ccecf7407b6c485983fd6ab15986d684 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/B/ccecf7407b6c485983fd6ab15986d684 2024-11-27T07:25:26,593 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/a90209be2e9748d9a433c1182fb1c83d to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/a90209be2e9748d9a433c1182fb1c83d 2024-11-27T07:25:26,595 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/bc35257bd5884e748285fbec85dfe306 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/C/bc35257bd5884e748285fbec85dfe306 2024-11-27T07:25:26,600 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/recovered.edits/426.seqid to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070/recovered.edits/426.seqid 2024-11-27T07:25:26,601 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:25:26,601 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-27T07:25:26,601 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-27T07:25:26,603 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-27T07:25:26,610 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127098043b48ce440fd8aa8c458345a4fae_225833644b1bebd5fa53ae861d7c7070 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127098043b48ce440fd8aa8c458345a4fae_225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:25:26,612 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411271e119778b7544994a69a28a716e54b69_225833644b1bebd5fa53ae861d7c7070 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411271e119778b7544994a69a28a716e54b69_225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:25:26,614 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411272dd0879ca8fe48c4a1bf987310c3819f_225833644b1bebd5fa53ae861d7c7070 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411272dd0879ca8fe48c4a1bf987310c3819f_225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:25:26,616 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112734c0f711b43b4310a964dd26288e6b3f_225833644b1bebd5fa53ae861d7c7070 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112734c0f711b43b4310a964dd26288e6b3f_225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:25:26,618 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127496b4dc66f83464095c1de12922bcbfc_225833644b1bebd5fa53ae861d7c7070 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127496b4dc66f83464095c1de12922bcbfc_225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:25:26,619 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411275010b4ff75224a1980743a11dc033ce6_225833644b1bebd5fa53ae861d7c7070 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411275010b4ff75224a1980743a11dc033ce6_225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:25:26,621 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411275ea753432bfe4e128832e53872ceaea0_225833644b1bebd5fa53ae861d7c7070 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411275ea753432bfe4e128832e53872ceaea0_225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:25:26,623 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112765a674e8ff33452ca5d79688591138af_225833644b1bebd5fa53ae861d7c7070 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112765a674e8ff33452ca5d79688591138af_225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:25:26,625 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127769ba03a5429477694366d8bf519c59e_225833644b1bebd5fa53ae861d7c7070 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127769ba03a5429477694366d8bf519c59e_225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:25:26,627 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411277cb2443f69fe436c8520d37d11029dde_225833644b1bebd5fa53ae861d7c7070 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411277cb2443f69fe436c8520d37d11029dde_225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:25:26,628 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411277e5351021b59446ab00728b7dbca8eb5_225833644b1bebd5fa53ae861d7c7070 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411277e5351021b59446ab00728b7dbca8eb5_225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:25:26,629 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112787115b489906465ba5448fa594c62530_225833644b1bebd5fa53ae861d7c7070 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112787115b489906465ba5448fa594c62530_225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:25:26,630 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411278ef2f316c910481cb5d5b657feaf364e_225833644b1bebd5fa53ae861d7c7070 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411278ef2f316c910481cb5d5b657feaf364e_225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:25:26,631 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112790837c8c25234d68988b53700547e560_225833644b1bebd5fa53ae861d7c7070 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112790837c8c25234d68988b53700547e560_225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:25:26,632 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127a1cd0fa243ec486382ed0ae3e1b88ba8_225833644b1bebd5fa53ae861d7c7070 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127a1cd0fa243ec486382ed0ae3e1b88ba8_225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:25:26,634 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127a6aaf13a5e09436884f4be45804f645e_225833644b1bebd5fa53ae861d7c7070 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127a6aaf13a5e09436884f4be45804f645e_225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:25:26,635 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127ba42c86a2f1f485e80580a759164eed4_225833644b1bebd5fa53ae861d7c7070 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127ba42c86a2f1f485e80580a759164eed4_225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:25:26,636 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127bfe07aa57b754b25939a5944d488e2c1_225833644b1bebd5fa53ae861d7c7070 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127bfe07aa57b754b25939a5944d488e2c1_225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:25:26,636 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127c859b2626c6b40c0b178c95b2e822acc_225833644b1bebd5fa53ae861d7c7070 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127c859b2626c6b40c0b178c95b2e822acc_225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:25:26,637 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127ccba5d04f2024b558792120b72e0a4f0_225833644b1bebd5fa53ae861d7c7070 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127ccba5d04f2024b558792120b72e0a4f0_225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:25:26,638 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127cf61ce19df5e46848f429304f6fab0b3_225833644b1bebd5fa53ae861d7c7070 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127cf61ce19df5e46848f429304f6fab0b3_225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:25:26,639 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127f561096555b6484082bf4ad843a86c56_225833644b1bebd5fa53ae861d7c7070 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127f561096555b6484082bf4ad843a86c56_225833644b1bebd5fa53ae861d7c7070 2024-11-27T07:25:26,640 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-27T07:25:26,642 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=62, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T07:25:26,644 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-27T07:25:26,646 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-27T07:25:26,647 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=62, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T07:25:26,647 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-27T07:25:26,647 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732692326647"}]},"ts":"9223372036854775807"} 2024-11-27T07:25:26,649 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-27T07:25:26,650 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 225833644b1bebd5fa53ae861d7c7070, NAME => 'TestAcidGuarantees,,1732692292003.225833644b1bebd5fa53ae861d7c7070.', STARTKEY => '', ENDKEY => ''}] 2024-11-27T07:25:26,650 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-27T07:25:26,650 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732692326650"}]},"ts":"9223372036854775807"} 2024-11-27T07:25:26,652 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-27T07:25:26,652 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=62, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T07:25:26,654 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 90 msec 2024-11-27T07:25:26,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-27T07:25:26,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-27T07:25:26,669 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-27T07:25:26,679 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=238 (was 239), OpenFileDescriptor=457 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=477 (was 507), ProcessCount=11 (was 11), AvailableMemoryMB=4386 (was 4784) 2024-11-27T07:25:26,690 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=238, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=477, ProcessCount=11, AvailableMemoryMB=4386 2024-11-27T07:25:26,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-27T07:25:26,692 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T07:25:26,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] procedure2.ProcedureExecutor(1139): Stored pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-27T07:25:26,694 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T07:25:26,694 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:25:26,694 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 63 2024-11-27T07:25:26,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-27T07:25:26,695 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T07:25:26,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742063_1239 (size=959) 2024-11-27T07:25:26,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-27T07:25:27,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-27T07:25:27,106 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4 2024-11-27T07:25:27,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742064_1240 (size=53) 2024-11-27T07:25:27,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-27T07:25:27,517 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T07:25:27,517 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 81ee2c488a54a955269ce99517372c4b, disabling compactions & flushes 2024-11-27T07:25:27,517 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:27,517 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:27,517 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. after waiting 0 ms 2024-11-27T07:25:27,517 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:27,517 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:27,517 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 81ee2c488a54a955269ce99517372c4b: Waiting for close lock at 1732692327517Disabling compacts and flushes for region at 1732692327517Disabling writes for close at 1732692327517Writing region close event to WAL at 1732692327517Closed at 1732692327517 2024-11-27T07:25:27,518 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T07:25:27,518 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732692327518"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732692327518"}]},"ts":"1732692327518"} 2024-11-27T07:25:27,520 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T07:25:27,521 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T07:25:27,521 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732692327521"}]},"ts":"1732692327521"} 2024-11-27T07:25:27,522 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-27T07:25:27,522 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=81ee2c488a54a955269ce99517372c4b, ASSIGN}] 2024-11-27T07:25:27,523 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=81ee2c488a54a955269ce99517372c4b, ASSIGN 2024-11-27T07:25:27,524 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=81ee2c488a54a955269ce99517372c4b, ASSIGN; state=OFFLINE, location=289954c9202d,42117,1732692259671; forceNewPlan=false, retain=false 2024-11-27T07:25:27,676 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=81ee2c488a54a955269ce99517372c4b, regionState=OPENING, regionLocation=289954c9202d,42117,1732692259671 2024-11-27T07:25:27,680 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=81ee2c488a54a955269ce99517372c4b, ASSIGN because future has completed 2024-11-27T07:25:27,681 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671}] 2024-11-27T07:25:27,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-27T07:25:27,844 INFO [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:27,844 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7752): Opening region: {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} 2024-11-27T07:25:27,844 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 81ee2c488a54a955269ce99517372c4b 2024-11-27T07:25:27,845 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T07:25:27,845 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7794): checking encryption for 81ee2c488a54a955269ce99517372c4b 2024-11-27T07:25:27,845 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7797): checking classloading for 81ee2c488a54a955269ce99517372c4b 2024-11-27T07:25:27,846 INFO [StoreOpener-81ee2c488a54a955269ce99517372c4b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 81ee2c488a54a955269ce99517372c4b 2024-11-27T07:25:27,848 INFO [StoreOpener-81ee2c488a54a955269ce99517372c4b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 81ee2c488a54a955269ce99517372c4b columnFamilyName A 2024-11-27T07:25:27,848 DEBUG [StoreOpener-81ee2c488a54a955269ce99517372c4b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:25:27,849 INFO [StoreOpener-81ee2c488a54a955269ce99517372c4b-1 {}] regionserver.HStore(327): Store=81ee2c488a54a955269ce99517372c4b/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T07:25:27,849 INFO [StoreOpener-81ee2c488a54a955269ce99517372c4b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 81ee2c488a54a955269ce99517372c4b 2024-11-27T07:25:27,850 INFO [StoreOpener-81ee2c488a54a955269ce99517372c4b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 81ee2c488a54a955269ce99517372c4b columnFamilyName B 2024-11-27T07:25:27,850 DEBUG [StoreOpener-81ee2c488a54a955269ce99517372c4b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:25:27,851 INFO [StoreOpener-81ee2c488a54a955269ce99517372c4b-1 {}] regionserver.HStore(327): Store=81ee2c488a54a955269ce99517372c4b/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T07:25:27,851 INFO [StoreOpener-81ee2c488a54a955269ce99517372c4b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 81ee2c488a54a955269ce99517372c4b 2024-11-27T07:25:27,853 INFO [StoreOpener-81ee2c488a54a955269ce99517372c4b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 81ee2c488a54a955269ce99517372c4b columnFamilyName C 2024-11-27T07:25:27,853 DEBUG [StoreOpener-81ee2c488a54a955269ce99517372c4b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:25:27,853 INFO [StoreOpener-81ee2c488a54a955269ce99517372c4b-1 {}] regionserver.HStore(327): Store=81ee2c488a54a955269ce99517372c4b/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T07:25:27,853 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1038): replaying wal for 81ee2c488a54a955269ce99517372c4b 2024-11-27T07:25:27,854 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b 2024-11-27T07:25:27,854 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b 2024-11-27T07:25:27,856 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1048): stopping wal replay for 81ee2c488a54a955269ce99517372c4b 2024-11-27T07:25:27,856 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1060): Cleaning up temporary data for 81ee2c488a54a955269ce99517372c4b 2024-11-27T07:25:27,856 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T07:25:27,857 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1093): writing seq id for 81ee2c488a54a955269ce99517372c4b 2024-11-27T07:25:27,859 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T07:25:27,860 INFO [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1114): Opened 81ee2c488a54a955269ce99517372c4b; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66045027, jitterRate=-0.01585240662097931}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T07:25:27,860 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 81ee2c488a54a955269ce99517372c4b 2024-11-27T07:25:27,860 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1006): Region open journal for 81ee2c488a54a955269ce99517372c4b: Running coprocessor pre-open hook at 1732692327845Writing region info on filesystem at 1732692327845Initializing all the Stores at 1732692327846 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732692327846Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732692327846Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732692327846Cleaning up temporary data from old regions at 1732692327856 (+10 ms)Running coprocessor post-open hooks at 1732692327860 (+4 ms)Region opened successfully at 1732692327860 2024-11-27T07:25:27,861 INFO [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., pid=65, masterSystemTime=1732692327835 2024-11-27T07:25:27,863 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:27,863 INFO [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:27,863 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=81ee2c488a54a955269ce99517372c4b, regionState=OPEN, openSeqNum=2, regionLocation=289954c9202d,42117,1732692259671 2024-11-27T07:25:27,864 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 because future has completed 2024-11-27T07:25:27,866 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-27T07:25:27,866 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; OpenRegionProcedure 81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 in 184 msec 2024-11-27T07:25:27,868 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-11-27T07:25:27,868 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=81ee2c488a54a955269ce99517372c4b, ASSIGN in 344 msec 2024-11-27T07:25:27,869 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T07:25:27,869 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732692327869"}]},"ts":"1732692327869"} 2024-11-27T07:25:27,870 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-27T07:25:27,871 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T07:25:27,872 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1790 sec 2024-11-27T07:25:28,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-27T07:25:28,840 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-27T07:25:28,841 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@333e91c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:25:28,841 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 289954c9202d,43057,-1 for getting cluster id 2024-11-27T07:25:28,841 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T07:25:28,842 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '09f43d3f-c175-4713-a9ce-038f5695b74b' 2024-11-27T07:25:28,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T07:25:28,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "09f43d3f-c175-4713-a9ce-038f5695b74b" 2024-11-27T07:25:28,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cfbd4c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:25:28,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [289954c9202d,43057,-1] 2024-11-27T07:25:28,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T07:25:28,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:25:28,844 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57622, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T07:25:28,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10a5339c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:25:28,845 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T07:25:28,846 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=289954c9202d,42117,1732692259671, seqNum=-1] 2024-11-27T07:25:28,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T07:25:28,848 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59256, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T07:25:28,849 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-27T07:25:28,849 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 289954c9202d,43057,1732692258941 2024-11-27T07:25:28,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4b603127 2024-11-27T07:25:28,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-27T07:25:28,851 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57638, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-27T07:25:28,853 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25146acd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:25:28,853 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 289954c9202d,43057,-1 for getting cluster id 2024-11-27T07:25:28,853 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T07:25:28,854 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '09f43d3f-c175-4713-a9ce-038f5695b74b' 2024-11-27T07:25:28,854 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T07:25:28,854 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "09f43d3f-c175-4713-a9ce-038f5695b74b" 2024-11-27T07:25:28,854 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ae56015, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:25:28,854 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [289954c9202d,43057,-1] 2024-11-27T07:25:28,855 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T07:25:28,855 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:25:28,856 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57664, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T07:25:28,857 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@690fc41d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:25:28,857 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@391cff66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:25:28,858 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 289954c9202d,43057,-1 for getting cluster id 2024-11-27T07:25:28,858 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T07:25:28,864 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '09f43d3f-c175-4713-a9ce-038f5695b74b' 2024-11-27T07:25:28,864 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T07:25:28,864 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "09f43d3f-c175-4713-a9ce-038f5695b74b" 2024-11-27T07:25:28,864 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31aa940, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:25:28,864 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [289954c9202d,43057,-1] 2024-11-27T07:25:28,865 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T07:25:28,865 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:25:28,866 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57688, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T07:25:28,866 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5493c1e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:25:28,867 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@135bd0ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:25:28,867 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 289954c9202d,43057,-1 for getting cluster id 2024-11-27T07:25:28,867 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T07:25:28,868 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '09f43d3f-c175-4713-a9ce-038f5695b74b' 2024-11-27T07:25:28,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T07:25:28,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "09f43d3f-c175-4713-a9ce-038f5695b74b" 2024-11-27T07:25:28,869 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63de9ae0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:25:28,869 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [289954c9202d,43057,-1] 2024-11-27T07:25:28,869 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T07:25:28,869 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:25:28,870 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57708, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T07:25:28,871 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@edce638, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:25:28,872 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4609cead, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:25:28,872 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 289954c9202d,43057,-1 for getting cluster id 2024-11-27T07:25:28,872 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T07:25:28,873 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '09f43d3f-c175-4713-a9ce-038f5695b74b' 2024-11-27T07:25:28,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T07:25:28,874 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "09f43d3f-c175-4713-a9ce-038f5695b74b" 2024-11-27T07:25:28,874 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e3cf3f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:25:28,874 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [289954c9202d,43057,-1] 2024-11-27T07:25:28,874 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T07:25:28,874 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:25:28,875 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57716, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T07:25:28,876 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16ebf716, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:25:28,877 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a2df097, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:25:28,878 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 289954c9202d,43057,-1 for getting cluster id 2024-11-27T07:25:28,878 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T07:25:28,879 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '09f43d3f-c175-4713-a9ce-038f5695b74b' 2024-11-27T07:25:28,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T07:25:28,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "09f43d3f-c175-4713-a9ce-038f5695b74b" 2024-11-27T07:25:28,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3099f549, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:25:28,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [289954c9202d,43057,-1] 2024-11-27T07:25:28,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T07:25:28,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:25:28,881 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57730, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T07:25:28,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c0632fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:25:28,884 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5559acf0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:25:28,884 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 289954c9202d,43057,-1 for getting cluster id 2024-11-27T07:25:28,884 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T07:25:28,885 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '09f43d3f-c175-4713-a9ce-038f5695b74b' 2024-11-27T07:25:28,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T07:25:28,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "09f43d3f-c175-4713-a9ce-038f5695b74b" 2024-11-27T07:25:28,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24852535, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:25:28,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [289954c9202d,43057,-1] 2024-11-27T07:25:28,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T07:25:28,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:25:28,887 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57740, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T07:25:28,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f34c93c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:25:28,894 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a4f6020, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:25:28,894 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 289954c9202d,43057,-1 for getting cluster id 2024-11-27T07:25:28,895 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T07:25:28,896 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '09f43d3f-c175-4713-a9ce-038f5695b74b' 2024-11-27T07:25:28,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T07:25:28,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "09f43d3f-c175-4713-a9ce-038f5695b74b" 2024-11-27T07:25:28,897 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:25:28,897 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b9635b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:25:28,897 DEBUG [Time-limited test {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [289954c9202d,43057,-1] 2024-11-27T07:25:28,898 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T07:25:28,899 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57756, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T07:25:28,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2952f5b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:25:28,903 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bee866e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:25:28,903 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 289954c9202d,43057,-1 for getting cluster id 2024-11-27T07:25:28,904 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T07:25:28,905 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '09f43d3f-c175-4713-a9ce-038f5695b74b' 2024-11-27T07:25:28,905 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T07:25:28,905 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "09f43d3f-c175-4713-a9ce-038f5695b74b" 2024-11-27T07:25:28,905 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7aeb6b4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:25:28,905 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [289954c9202d,43057,-1] 2024-11-27T07:25:28,906 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T07:25:28,906 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:25:28,907 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57784, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T07:25:28,908 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53fa97c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:25:28,912 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@148ac527, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:25:28,912 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 289954c9202d,43057,-1 for getting cluster id 2024-11-27T07:25:28,913 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T07:25:28,914 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '09f43d3f-c175-4713-a9ce-038f5695b74b' 2024-11-27T07:25:28,914 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T07:25:28,914 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "09f43d3f-c175-4713-a9ce-038f5695b74b" 2024-11-27T07:25:28,914 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3688a16b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:25:28,914 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [289954c9202d,43057,-1] 2024-11-27T07:25:28,914 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T07:25:28,914 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:25:28,915 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57796, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T07:25:28,916 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@603a650c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:25:28,917 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4199deaf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:25:28,917 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 289954c9202d,43057,-1 for getting cluster id 2024-11-27T07:25:28,917 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T07:25:28,918 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '09f43d3f-c175-4713-a9ce-038f5695b74b' 2024-11-27T07:25:28,918 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T07:25:28,918 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "09f43d3f-c175-4713-a9ce-038f5695b74b" 2024-11-27T07:25:28,918 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@279bd8ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:25:28,918 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [289954c9202d,43057,-1] 2024-11-27T07:25:28,918 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T07:25:28,919 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:25:28,919 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57808, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T07:25:28,920 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f0f776c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:25:28,921 DEBUG [Thread-1121 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T07:25:28,921 DEBUG [Thread-1140 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T07:25:28,922 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=289954c9202d,42117,1732692259671, seqNum=-1] 2024-11-27T07:25:28,922 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=289954c9202d,42117,1732692259671, seqNum=-1] 2024-11-27T07:25:28,922 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T07:25:28,922 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T07:25:28,923 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59266, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T07:25:28,924 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59268, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T07:25:28,925 DEBUG [Thread-1132 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T07:25:28,925 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2] 2024-11-27T07:25:28,925 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2] 2024-11-27T07:25:28,926 DEBUG [Thread-1138 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T07:25:28,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=289954c9202d,42117,1732692259671, seqNum=-1] 2024-11-27T07:25:28,927 DEBUG [Thread-1136 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T07:25:28,927 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=289954c9202d,42117,1732692259671, seqNum=-1] 2024-11-27T07:25:28,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T07:25:28,927 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T07:25:28,928 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=289954c9202d,42117,1732692259671, seqNum=-1] 2024-11-27T07:25:28,928 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T07:25:28,928 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59282, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T07:25:28,929 DEBUG [Thread-1125 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T07:25:28,930 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59296, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T07:25:28,930 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59298, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T07:25:28,930 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2] 2024-11-27T07:25:28,930 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=289954c9202d,42117,1732692259671, seqNum=-1] 2024-11-27T07:25:28,930 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T07:25:28,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2] 2024-11-27T07:25:28,931 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59308, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T07:25:28,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2] 2024-11-27T07:25:28,932 DEBUG [Thread-1134 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T07:25:28,934 DEBUG [Thread-1134 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=289954c9202d,42117,1732692259671, seqNum=-1] 2024-11-27T07:25:28,934 DEBUG [Thread-1134 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T07:25:28,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2] 2024-11-27T07:25:28,936 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59316, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T07:25:28,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2] 2024-11-27T07:25:28,952 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T07:25:28,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-27T07:25:28,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-27T07:25:28,955 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T07:25:28,955 DEBUG [Thread-1123 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T07:25:28,956 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T07:25:28,956 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=289954c9202d,42117,1732692259671, seqNum=-1] 2024-11-27T07:25:28,956 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T07:25:28,956 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T07:25:28,957 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59326, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T07:25:28,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 81ee2c488a54a955269ce99517372c4b 2024-11-27T07:25:28,958 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T07:25:28,958 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81ee2c488a54a955269ce99517372c4b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-27T07:25:28,959 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2] 2024-11-27T07:25:28,968 DEBUG [Thread-1127 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T07:25:28,969 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=289954c9202d,42117,1732692259671, seqNum=-1] 2024-11-27T07:25:28,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T07:25:28,971 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59334, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T07:25:28,973 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2] 2024-11-27T07:25:28,986 DEBUG [Thread-1129 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T07:25:28,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=289954c9202d,42117,1732692259671, seqNum=-1] 2024-11-27T07:25:28,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T07:25:28,996 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59340, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T07:25:28,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2] 2024-11-27T07:25:29,009 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/144330b3b9554335b20af695f36aa810 is 50, key is test_row_0/A:col10/1732692328954/Put/seqid=0 2024-11-27T07:25:29,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742065_1241 (size=14341) 2024-11-27T07:25:29,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:29,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59268 deadline: 1732692389032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:29,044 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:29,044 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:29,044 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:29,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:29,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59340 deadline: 1732692389038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:29,046 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:29,046 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:29,046 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:29,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:29,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1732692389039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:29,055 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:29,055 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:29,055 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:29,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:29,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59334 deadline: 1732692389045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:29,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-27T07:25:29,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:29,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:29,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:29,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:29,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59326 deadline: 1732692389051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:29,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:29,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:29,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:29,110 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-27T07:25:29,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:29,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. as already flushing 2024-11-27T07:25:29,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:29,110 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:29,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:29,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:29,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:29,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:29,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59340 deadline: 1732692389150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:29,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59268 deadline: 1732692389150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:29,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:29,154 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:29,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:29,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:29,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:29,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:29,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:29,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1732692389160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:29,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:29,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:29,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:29,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:29,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59334 deadline: 1732692389169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:29,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:29,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:29,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:29,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:29,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59326 deadline: 1732692389170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:29,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:29,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:29,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:29,264 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-27T07:25:29,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:29,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. as already flushing 2024-11-27T07:25:29,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:29,264 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:29,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:29,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:29,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-27T07:25:29,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:29,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59340 deadline: 1732692389359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:29,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:29,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:29,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:29,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:29,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59268 deadline: 1732692389359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:29,361 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:29,361 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:29,361 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:29,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:29,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1732692389369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:29,373 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:29,373 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:29,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:29,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:29,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59334 deadline: 1732692389379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:29,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:29,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:29,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:29,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:29,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59326 deadline: 1732692389380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:29,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:29,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:29,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:29,417 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-27T07:25:29,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:29,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. as already flushing 2024-11-27T07:25:29,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:29,418 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:29,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:29,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:29,436 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/144330b3b9554335b20af695f36aa810 2024-11-27T07:25:29,503 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/38564da6bc1044918d34852e6e66e773 is 50, key is test_row_0/B:col10/1732692328954/Put/seqid=0 2024-11-27T07:25:29,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742066_1242 (size=12001) 2024-11-27T07:25:29,532 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/38564da6bc1044918d34852e6e66e773 2024-11-27T07:25:29,570 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-27T07:25:29,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:29,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. as already flushing 2024-11-27T07:25:29,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:29,571 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:29,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:29,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:29,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-27T07:25:29,640 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/b7f2e755f27a45be8b2fbd609bcbdb27 is 50, key is test_row_0/C:col10/1732692328954/Put/seqid=0 2024-11-27T07:25:29,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742067_1243 (size=12001) 2024-11-27T07:25:29,657 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/b7f2e755f27a45be8b2fbd609bcbdb27 2024-11-27T07:25:29,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/144330b3b9554335b20af695f36aa810 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/144330b3b9554335b20af695f36aa810 2024-11-27T07:25:29,670 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/144330b3b9554335b20af695f36aa810, entries=200, sequenceid=13, filesize=14.0 K 2024-11-27T07:25:29,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/38564da6bc1044918d34852e6e66e773 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/38564da6bc1044918d34852e6e66e773 2024-11-27T07:25:29,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:29,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59340 deadline: 1732692389669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:29,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:29,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:29,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:29,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:29,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59268 deadline: 1732692389669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:29,674 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:29,674 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:29,674 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:29,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:29,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1732692389679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:29,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:29,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:29,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:29,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:29,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:29,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59334 deadline: 1732692389689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:29,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59326 deadline: 1732692389689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:29,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:29,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:29,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:29,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:29,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:29,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:29,719 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/38564da6bc1044918d34852e6e66e773, entries=150, sequenceid=13, filesize=11.7 K 2024-11-27T07:25:29,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/b7f2e755f27a45be8b2fbd609bcbdb27 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/b7f2e755f27a45be8b2fbd609bcbdb27 2024-11-27T07:25:29,728 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/b7f2e755f27a45be8b2fbd609bcbdb27, entries=150, sequenceid=13, filesize=11.7 K 2024-11-27T07:25:29,728 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-27T07:25:29,729 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 81ee2c488a54a955269ce99517372c4b in 771ms, sequenceid=13, compaction requested=false 2024-11-27T07:25:29,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81ee2c488a54a955269ce99517372c4b: 2024-11-27T07:25:29,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:29,729 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 81ee2c488a54a955269ce99517372c4b 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-27T07:25:29,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/14f86348aa704449b4d7cac0a78c97c1 is 50, key is test_row_0/A:col10/1732692329040/Put/seqid=0 2024-11-27T07:25:29,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742068_1244 (size=12001) 2024-11-27T07:25:29,765 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/14f86348aa704449b4d7cac0a78c97c1 2024-11-27T07:25:29,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/1ca2d866ad474e18a09010c44232404f is 50, key is test_row_0/B:col10/1732692329040/Put/seqid=0 2024-11-27T07:25:29,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742069_1245 (size=12001) 2024-11-27T07:25:29,806 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/1ca2d866ad474e18a09010c44232404f 2024-11-27T07:25:29,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/78698476588646b38f69c7bcca09e1c2 is 50, key is test_row_0/C:col10/1732692329040/Put/seqid=0 2024-11-27T07:25:29,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742070_1246 (size=12001) 2024-11-27T07:25:29,840 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/78698476588646b38f69c7bcca09e1c2 2024-11-27T07:25:29,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/14f86348aa704449b4d7cac0a78c97c1 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/14f86348aa704449b4d7cac0a78c97c1 2024-11-27T07:25:29,850 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/14f86348aa704449b4d7cac0a78c97c1, entries=150, sequenceid=38, filesize=11.7 K 2024-11-27T07:25:29,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/1ca2d866ad474e18a09010c44232404f as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/1ca2d866ad474e18a09010c44232404f 2024-11-27T07:25:29,857 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/1ca2d866ad474e18a09010c44232404f, entries=150, sequenceid=38, filesize=11.7 K 2024-11-27T07:25:29,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/78698476588646b38f69c7bcca09e1c2 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/78698476588646b38f69c7bcca09e1c2 2024-11-27T07:25:29,863 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/78698476588646b38f69c7bcca09e1c2, entries=150, sequenceid=38, filesize=11.7 K 2024-11-27T07:25:29,865 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=0 B/0 for 81ee2c488a54a955269ce99517372c4b in 135ms, sequenceid=38, compaction requested=false 2024-11-27T07:25:29,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 81ee2c488a54a955269ce99517372c4b: 2024-11-27T07:25:29,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:29,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-27T07:25:29,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-27T07:25:29,871 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-27T07:25:29,871 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 912 msec 2024-11-27T07:25:29,874 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 920 msec 2024-11-27T07:25:30,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-27T07:25:30,130 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T07:25:30,130 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T07:25:30,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-27T07:25:30,137 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T07:25:30,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-27T07:25:30,138 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T07:25:30,138 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T07:25:30,199 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T07:25:30,199 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81ee2c488a54a955269ce99517372c4b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-27T07:25:30,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 81ee2c488a54a955269ce99517372c4b 2024-11-27T07:25:30,216 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/3adf4dd86d674f4fa37cb467e6ea44ca is 50, key is test_row_0/A:col10/1732692330191/Put/seqid=0 2024-11-27T07:25:30,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-27T07:25:30,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742071_1247 (size=12001) 2024-11-27T07:25:30,250 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/3adf4dd86d674f4fa37cb467e6ea44ca 2024-11-27T07:25:30,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:30,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59334 deadline: 1732692390251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:30,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:30,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:30,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:30,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:30,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59340 deadline: 1732692390258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:30,263 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:30,263 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:30,263 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:30,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:30,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1732692390259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:30,264 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:30,264 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:30,264 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:30,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:30,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59326 deadline: 1732692390261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:30,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:30,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59268 deadline: 1732692390261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:30,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:30,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:30,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:30,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:30,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:30,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:30,290 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/4c0b025b161d4a1386b1d6f16ca9a039 is 50, key is test_row_0/B:col10/1732692330191/Put/seqid=0 2024-11-27T07:25:30,294 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-27T07:25:30,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:30,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. as already flushing 2024-11-27T07:25:30,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:30,295 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:30,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:30,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:30,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742072_1248 (size=12001) 2024-11-27T07:25:30,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:30,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59334 deadline: 1732692390369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:30,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:30,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1732692390369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:30,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:30,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:30,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:30,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59340 deadline: 1732692390369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:30,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:30,372 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:30,372 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:30,372 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:30,373 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:30,373 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:30,373 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:30,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:30,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59268 deadline: 1732692390379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:30,382 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:30,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:30,382 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:30,382 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:30,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59326 deadline: 1732692390379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:30,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:30,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:30,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:30,448 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-27T07:25:30,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:30,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. as already flushing 2024-11-27T07:25:30,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:30,448 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:30,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:30,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-27T07:25:30,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:30,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:30,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59334 deadline: 1732692390579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:30,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:30,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59340 deadline: 1732692390579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:30,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:30,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:30,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:30,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:30,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1732692390579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:30,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:30,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:30,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:30,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:30,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:30,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:30,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:30,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59268 deadline: 1732692390589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:30,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:30,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:30,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:30,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:30,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59326 deadline: 1732692390589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:30,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:30,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:30,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:30,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-27T07:25:30,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:30,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. as already flushing 2024-11-27T07:25:30,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:30,603 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:30,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:30,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:30,743 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/4c0b025b161d4a1386b1d6f16ca9a039 2024-11-27T07:25:30,754 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/cdec4d8b0b70426fa32dc011a9f64ef4 is 50, key is test_row_0/C:col10/1732692330191/Put/seqid=0 2024-11-27T07:25:30,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-27T07:25:30,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:30,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. as already flushing 2024-11-27T07:25:30,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:30,756 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:30,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:30,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:30,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742073_1249 (size=12001) 2024-11-27T07:25:30,759 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/cdec4d8b0b70426fa32dc011a9f64ef4 2024-11-27T07:25:30,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-27T07:25:30,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/3adf4dd86d674f4fa37cb467e6ea44ca as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/3adf4dd86d674f4fa37cb467e6ea44ca 2024-11-27T07:25:30,771 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/3adf4dd86d674f4fa37cb467e6ea44ca, entries=150, sequenceid=50, filesize=11.7 K 2024-11-27T07:25:30,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/4c0b025b161d4a1386b1d6f16ca9a039 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/4c0b025b161d4a1386b1d6f16ca9a039 2024-11-27T07:25:30,777 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/4c0b025b161d4a1386b1d6f16ca9a039, entries=150, sequenceid=50, filesize=11.7 K 2024-11-27T07:25:30,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/cdec4d8b0b70426fa32dc011a9f64ef4 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/cdec4d8b0b70426fa32dc011a9f64ef4 2024-11-27T07:25:30,781 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/cdec4d8b0b70426fa32dc011a9f64ef4, entries=150, sequenceid=50, filesize=11.7 K 2024-11-27T07:25:30,782 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 81ee2c488a54a955269ce99517372c4b in 583ms, sequenceid=50, compaction requested=true 2024-11-27T07:25:30,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81ee2c488a54a955269ce99517372c4b: 2024-11-27T07:25:30,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81ee2c488a54a955269ce99517372c4b:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T07:25:30,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:25:30,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81ee2c488a54a955269ce99517372c4b:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T07:25:30,783 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:25:30,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:25:30,783 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:25:30,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81ee2c488a54a955269ce99517372c4b:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T07:25:30,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:25:30,783 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:25:30,784 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1541): 81ee2c488a54a955269ce99517372c4b/A is initiating minor compaction (all files) 2024-11-27T07:25:30,784 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81ee2c488a54a955269ce99517372c4b/A in TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:30,784 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/144330b3b9554335b20af695f36aa810, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/14f86348aa704449b4d7cac0a78c97c1, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/3adf4dd86d674f4fa37cb467e6ea44ca] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp, totalSize=37.4 K 2024-11-27T07:25:30,784 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:25:30,784 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 81ee2c488a54a955269ce99517372c4b/B is initiating minor compaction (all files) 2024-11-27T07:25:30,784 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81ee2c488a54a955269ce99517372c4b/B in TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:30,784 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/38564da6bc1044918d34852e6e66e773, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/1ca2d866ad474e18a09010c44232404f, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/4c0b025b161d4a1386b1d6f16ca9a039] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp, totalSize=35.2 K 2024-11-27T07:25:30,784 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 144330b3b9554335b20af695f36aa810, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732692328943 2024-11-27T07:25:30,785 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 38564da6bc1044918d34852e6e66e773, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732692328950 2024-11-27T07:25:30,785 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14f86348aa704449b4d7cac0a78c97c1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732692328984 2024-11-27T07:25:30,785 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ca2d866ad474e18a09010c44232404f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732692328984 2024-11-27T07:25:30,785 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3adf4dd86d674f4fa37cb467e6ea44ca, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732692330187 2024-11-27T07:25:30,785 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c0b025b161d4a1386b1d6f16ca9a039, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732692330187 2024-11-27T07:25:30,793 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81ee2c488a54a955269ce99517372c4b#B#compaction#201 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:25:30,793 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81ee2c488a54a955269ce99517372c4b#A#compaction#202 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:25:30,793 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/3eb82172993e4559ac4b18919d940945 is 50, key is test_row_0/B:col10/1732692330191/Put/seqid=0 2024-11-27T07:25:30,793 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/89e21cbcf5014318aa5bd91f194947b7 is 50, key is test_row_0/A:col10/1732692330191/Put/seqid=0 2024-11-27T07:25:30,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742074_1250 (size=12104) 2024-11-27T07:25:30,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742075_1251 (size=12104) 2024-11-27T07:25:30,804 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/89e21cbcf5014318aa5bd91f194947b7 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/89e21cbcf5014318aa5bd91f194947b7 2024-11-27T07:25:30,810 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81ee2c488a54a955269ce99517372c4b/A of 81ee2c488a54a955269ce99517372c4b into 89e21cbcf5014318aa5bd91f194947b7(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:25:30,810 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81ee2c488a54a955269ce99517372c4b: 2024-11-27T07:25:30,810 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., storeName=81ee2c488a54a955269ce99517372c4b/A, priority=13, startTime=1732692330782; duration=0sec 2024-11-27T07:25:30,811 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:25:30,811 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81ee2c488a54a955269ce99517372c4b:A 2024-11-27T07:25:30,811 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:25:30,812 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:25:30,812 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1541): 81ee2c488a54a955269ce99517372c4b/C is initiating minor compaction (all files) 2024-11-27T07:25:30,812 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81ee2c488a54a955269ce99517372c4b/C in TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:30,812 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/b7f2e755f27a45be8b2fbd609bcbdb27, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/78698476588646b38f69c7bcca09e1c2, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/cdec4d8b0b70426fa32dc011a9f64ef4] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp, totalSize=35.2 K 2024-11-27T07:25:30,812 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7f2e755f27a45be8b2fbd609bcbdb27, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732692328950 2024-11-27T07:25:30,813 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78698476588646b38f69c7bcca09e1c2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732692328984 2024-11-27T07:25:30,814 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting cdec4d8b0b70426fa32dc011a9f64ef4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732692330187 2024-11-27T07:25:30,821 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81ee2c488a54a955269ce99517372c4b#C#compaction#203 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:25:30,821 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/6416c66c71804937a58cca053720bfc2 is 50, key is test_row_0/C:col10/1732692330191/Put/seqid=0 2024-11-27T07:25:30,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742076_1252 (size=12104) 2024-11-27T07:25:30,831 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/6416c66c71804937a58cca053720bfc2 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/6416c66c71804937a58cca053720bfc2 2024-11-27T07:25:30,836 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81ee2c488a54a955269ce99517372c4b/C of 81ee2c488a54a955269ce99517372c4b into 6416c66c71804937a58cca053720bfc2(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:25:30,836 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81ee2c488a54a955269ce99517372c4b: 2024-11-27T07:25:30,836 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., storeName=81ee2c488a54a955269ce99517372c4b/C, priority=13, startTime=1732692330783; duration=0sec 2024-11-27T07:25:30,836 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:25:30,836 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81ee2c488a54a955269ce99517372c4b:C 2024-11-27T07:25:30,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 81ee2c488a54a955269ce99517372c4b 2024-11-27T07:25:30,892 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T07:25:30,893 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81ee2c488a54a955269ce99517372c4b 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-27T07:25:30,909 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-27T07:25:30,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:30,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. as already flushing 2024-11-27T07:25:30,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:30,909 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:30,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:30,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:30,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:30,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1732692390905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:30,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:30,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59340 deadline: 1732692390905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:30,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:30,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:30,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:30,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:30,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:30,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:30,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:30,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59268 deadline: 1732692390906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:30,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:30,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:30,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:30,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59326 deadline: 1732692390906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:30,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:30,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:30,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:30,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:30,916 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/f2ce488fdf0f44ad897de95134ed3aa7 is 50, key is test_row_0/A:col10/1732692330891/Put/seqid=0 2024-11-27T07:25:30,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:30,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59334 deadline: 1732692390912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:30,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:30,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:30,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:30,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742077_1253 (size=12001) 2024-11-27T07:25:31,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:31,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59340 deadline: 1732692391020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:31,022 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:31,022 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:31,022 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:31,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:31,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1732692391020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:31,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:31,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59326 deadline: 1732692391020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:31,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:31,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:31,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:31,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:31,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59268 deadline: 1732692391020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:31,025 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:31,025 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:31,025 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:31,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:31,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:31,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:31,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:31,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59334 deadline: 1732692391029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:31,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:31,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:31,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:31,062 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-27T07:25:31,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:31,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. as already flushing 2024-11-27T07:25:31,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:31,062 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:31,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:31,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:31,081 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T07:25:31,206 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/3eb82172993e4559ac4b18919d940945 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/3eb82172993e4559ac4b18919d940945 2024-11-27T07:25:31,211 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81ee2c488a54a955269ce99517372c4b/B of 81ee2c488a54a955269ce99517372c4b into 3eb82172993e4559ac4b18919d940945(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:25:31,212 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81ee2c488a54a955269ce99517372c4b: 2024-11-27T07:25:31,212 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., storeName=81ee2c488a54a955269ce99517372c4b/B, priority=13, startTime=1732692330783; duration=0sec 2024-11-27T07:25:31,212 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:25:31,212 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81ee2c488a54a955269ce99517372c4b:B 2024-11-27T07:25:31,215 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-27T07:25:31,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:31,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. as already flushing 2024-11-27T07:25:31,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:31,215 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:31,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:31,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:31,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:31,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59340 deadline: 1732692391229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:31,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:31,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:31,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59326 deadline: 1732692391229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:31,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:31,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:31,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:31,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:31,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:31,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59268 deadline: 1732692391229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:31,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:31,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:31,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:31,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:31,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:31,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1732692391239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:31,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:31,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:31,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:31,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:31,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59334 deadline: 1732692391240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:31,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:31,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:31,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:31,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-27T07:25:31,367 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/f2ce488fdf0f44ad897de95134ed3aa7 2024-11-27T07:25:31,369 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-27T07:25:31,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:31,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. as already flushing 2024-11-27T07:25:31,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:31,370 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:31,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:31,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:31,388 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/f89f409dff994323990043c02ca3d15a is 50, key is test_row_0/B:col10/1732692330891/Put/seqid=0 2024-11-27T07:25:31,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742078_1254 (size=12001) 2024-11-27T07:25:31,429 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/f89f409dff994323990043c02ca3d15a 2024-11-27T07:25:31,443 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/002d440bf6a54d44854c3e4a3d5d6c37 is 50, key is test_row_0/C:col10/1732692330891/Put/seqid=0 2024-11-27T07:25:31,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742079_1255 (size=12001) 2024-11-27T07:25:31,491 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/002d440bf6a54d44854c3e4a3d5d6c37 2024-11-27T07:25:31,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/f2ce488fdf0f44ad897de95134ed3aa7 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/f2ce488fdf0f44ad897de95134ed3aa7 2024-11-27T07:25:31,508 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/f2ce488fdf0f44ad897de95134ed3aa7, entries=150, sequenceid=77, filesize=11.7 K 2024-11-27T07:25:31,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/f89f409dff994323990043c02ca3d15a as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/f89f409dff994323990043c02ca3d15a 2024-11-27T07:25:31,518 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/f89f409dff994323990043c02ca3d15a, entries=150, sequenceid=77, filesize=11.7 K 2024-11-27T07:25:31,520 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/002d440bf6a54d44854c3e4a3d5d6c37 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/002d440bf6a54d44854c3e4a3d5d6c37 2024-11-27T07:25:31,525 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/002d440bf6a54d44854c3e4a3d5d6c37, entries=150, sequenceid=77, filesize=11.7 K 2024-11-27T07:25:31,526 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 81ee2c488a54a955269ce99517372c4b in 634ms, sequenceid=77, compaction requested=false 2024-11-27T07:25:31,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81ee2c488a54a955269ce99517372c4b: 2024-11-27T07:25:31,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-27T07:25:31,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:31,529 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 81ee2c488a54a955269ce99517372c4b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-27T07:25:31,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. as already flushing 2024-11-27T07:25:31,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 81ee2c488a54a955269ce99517372c4b 2024-11-27T07:25:31,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/a735557a9fb348da889c980b985d97af is 50, key is test_row_0/A:col10/1732692330904/Put/seqid=0 2024-11-27T07:25:31,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742080_1256 (size=12001) 2024-11-27T07:25:31,595 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/a735557a9fb348da889c980b985d97af 2024-11-27T07:25:31,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:31,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59340 deadline: 1732692391588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:31,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:31,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1732692391590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:31,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:31,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:31,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:31,597 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:31,597 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:31,597 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:31,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:31,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59326 deadline: 1732692391591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:31,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:31,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:31,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:31,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:31,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59268 deadline: 1732692391592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:31,600 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:31,600 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:31,600 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:31,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:31,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59334 deadline: 1732692391594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:31,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:31,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:31,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:31,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/c71ae23fecaa4b8b9097ac0ec73e9e96 is 50, key is test_row_0/B:col10/1732692330904/Put/seqid=0 2024-11-27T07:25:31,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742081_1257 (size=12001) 2024-11-27T07:25:31,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:31,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:31,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59340 deadline: 1732692391699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:31,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1732692391699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:31,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:31,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:31,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:31,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:31,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:31,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:31,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:31,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:31,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59334 deadline: 1732692391709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:31,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59326 deadline: 1732692391709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:31,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:31,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59268 deadline: 1732692391709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:31,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:31,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:31,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:31,716 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:31,716 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:31,716 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:31,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:31,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:31,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:31,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:31,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59340 deadline: 1732692391909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:31,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:31,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:31,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:31,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:31,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1732692391910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:31,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:31,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:31,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:31,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:31,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59334 deadline: 1732692391920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:31,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:31,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59326 deadline: 1732692391920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:31,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:31,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:31,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59268 deadline: 1732692391920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:31,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:31,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:31,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:31,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:31,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:31,925 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:31,925 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:31,925 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:32,075 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/c71ae23fecaa4b8b9097ac0ec73e9e96 2024-11-27T07:25:32,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/c741face4aba407ca4f45e19870fcbbb is 50, key is test_row_0/C:col10/1732692330904/Put/seqid=0 2024-11-27T07:25:32,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742082_1258 (size=12001) 2024-11-27T07:25:32,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:32,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59340 deadline: 1732692392219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:32,222 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:32,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:32,222 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:32,222 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:32,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1732692392219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:32,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:32,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:32,223 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:32,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:32,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59326 deadline: 1732692392229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:32,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:32,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59268 deadline: 1732692392229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:32,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:32,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:32,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:32,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:32,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:32,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:32,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59334 deadline: 1732692392229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:32,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:32,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:32,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:32,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:32,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-27T07:25:32,490 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/c741face4aba407ca4f45e19870fcbbb 2024-11-27T07:25:32,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/a735557a9fb348da889c980b985d97af as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/a735557a9fb348da889c980b985d97af 2024-11-27T07:25:32,501 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/a735557a9fb348da889c980b985d97af, entries=150, sequenceid=89, filesize=11.7 K 2024-11-27T07:25:32,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/c71ae23fecaa4b8b9097ac0ec73e9e96 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/c71ae23fecaa4b8b9097ac0ec73e9e96 2024-11-27T07:25:32,510 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/c71ae23fecaa4b8b9097ac0ec73e9e96, entries=150, sequenceid=89, filesize=11.7 K 2024-11-27T07:25:32,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/c741face4aba407ca4f45e19870fcbbb as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/c741face4aba407ca4f45e19870fcbbb 2024-11-27T07:25:32,516 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/c741face4aba407ca4f45e19870fcbbb, entries=150, sequenceid=89, filesize=11.7 K 2024-11-27T07:25:32,517 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 81ee2c488a54a955269ce99517372c4b in 988ms, sequenceid=89, compaction requested=true 2024-11-27T07:25:32,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 81ee2c488a54a955269ce99517372c4b: 2024-11-27T07:25:32,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:32,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-27T07:25:32,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-27T07:25:32,521 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-27T07:25:32,521 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3810 sec 2024-11-27T07:25:32,523 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 2.3920 sec 2024-11-27T07:25:32,733 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T07:25:32,733 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81ee2c488a54a955269ce99517372c4b 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-27T07:25:32,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 81ee2c488a54a955269ce99517372c4b 2024-11-27T07:25:32,744 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/2ff90d4d98d94d59a9141423f91b2e21 is 50, key is test_row_0/A:col10/1732692332732/Put/seqid=0 2024-11-27T07:25:32,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742083_1259 (size=16681) 2024-11-27T07:25:32,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:32,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59268 deadline: 1732692392755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:32,811 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:32,811 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:32,811 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:32,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:32,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1732692392755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:32,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:32,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:32,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:32,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:32,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59334 deadline: 1732692392811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:32,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:32,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:32,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:32,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59326 deadline: 1732692392811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:32,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:32,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:32,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:32,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:32,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:32,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59340 deadline: 1732692392818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:32,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:32,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:32,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:32,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:32,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59268 deadline: 1732692392919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:32,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:32,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:32,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:32,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1732692392919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:32,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:32,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:32,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:32,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:32,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:32,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59334 deadline: 1732692392919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:32,923 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:32,923 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:32,923 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:32,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:32,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59326 deadline: 1732692392930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:32,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:32,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:32,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:32,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:32,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59340 deadline: 1732692392930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:32,934 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:32,934 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:32,934 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:33,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:33,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:33,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1732692393130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:33,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59268 deadline: 1732692393130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:33,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:33,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:33,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:33,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59334 deadline: 1732692393130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:33,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:33,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:33,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:33,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:33,135 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:33,135 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:33,135 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:33,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:33,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59326 deadline: 1732692393139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:33,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:33,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:33,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:33,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:33,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59340 deadline: 1732692393139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:33,143 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:33,143 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:33,143 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:33,170 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/2ff90d4d98d94d59a9141423f91b2e21 2024-11-27T07:25:33,198 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/302e213fd0cd4f1d8157d388bd82cf93 is 50, key is test_row_0/B:col10/1732692332732/Put/seqid=0 2024-11-27T07:25:33,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742084_1260 (size=12001) 2024-11-27T07:25:33,219 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/302e213fd0cd4f1d8157d388bd82cf93 2024-11-27T07:25:33,230 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/d8d757558f1f4b6f8eb6046b1ac3be87 is 50, key is test_row_0/C:col10/1732692332732/Put/seqid=0 2024-11-27T07:25:33,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742085_1261 (size=12001) 2024-11-27T07:25:33,255 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/d8d757558f1f4b6f8eb6046b1ac3be87 2024-11-27T07:25:33,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/2ff90d4d98d94d59a9141423f91b2e21 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/2ff90d4d98d94d59a9141423f91b2e21 2024-11-27T07:25:33,267 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/2ff90d4d98d94d59a9141423f91b2e21, entries=250, sequenceid=116, filesize=16.3 K 2024-11-27T07:25:33,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/302e213fd0cd4f1d8157d388bd82cf93 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/302e213fd0cd4f1d8157d388bd82cf93 2024-11-27T07:25:33,275 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/302e213fd0cd4f1d8157d388bd82cf93, entries=150, sequenceid=116, filesize=11.7 K 2024-11-27T07:25:33,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/d8d757558f1f4b6f8eb6046b1ac3be87 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/d8d757558f1f4b6f8eb6046b1ac3be87 2024-11-27T07:25:33,285 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/d8d757558f1f4b6f8eb6046b1ac3be87, entries=150, sequenceid=116, filesize=11.7 K 2024-11-27T07:25:33,286 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=53.67 KB/54960 for 81ee2c488a54a955269ce99517372c4b in 553ms, sequenceid=116, compaction requested=true 2024-11-27T07:25:33,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81ee2c488a54a955269ce99517372c4b: 2024-11-27T07:25:33,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81ee2c488a54a955269ce99517372c4b:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T07:25:33,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:25:33,287 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T07:25:33,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81ee2c488a54a955269ce99517372c4b:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T07:25:33,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:25:33,287 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T07:25:33,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81ee2c488a54a955269ce99517372c4b:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T07:25:33,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:25:33,289 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T07:25:33,289 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 81ee2c488a54a955269ce99517372c4b/B is initiating minor compaction (all files) 2024-11-27T07:25:33,289 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81ee2c488a54a955269ce99517372c4b/B in TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:33,289 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/3eb82172993e4559ac4b18919d940945, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/f89f409dff994323990043c02ca3d15a, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/c71ae23fecaa4b8b9097ac0ec73e9e96, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/302e213fd0cd4f1d8157d388bd82cf93] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp, totalSize=47.0 K 2024-11-27T07:25:33,289 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52787 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T07:25:33,289 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1541): 81ee2c488a54a955269ce99517372c4b/A is initiating minor compaction (all files) 2024-11-27T07:25:33,289 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81ee2c488a54a955269ce99517372c4b/A in TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:33,290 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/89e21cbcf5014318aa5bd91f194947b7, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/f2ce488fdf0f44ad897de95134ed3aa7, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/a735557a9fb348da889c980b985d97af, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/2ff90d4d98d94d59a9141423f91b2e21] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp, totalSize=51.5 K 2024-11-27T07:25:33,290 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 3eb82172993e4559ac4b18919d940945, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732692330187 2024-11-27T07:25:33,290 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89e21cbcf5014318aa5bd91f194947b7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732692330187 2024-11-27T07:25:33,291 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2ce488fdf0f44ad897de95134ed3aa7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732692330246 2024-11-27T07:25:33,291 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting f89f409dff994323990043c02ca3d15a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732692330246 2024-11-27T07:25:33,291 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting a735557a9fb348da889c980b985d97af, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732692330900 2024-11-27T07:25:33,292 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting c71ae23fecaa4b8b9097ac0ec73e9e96, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732692330900 2024-11-27T07:25:33,301 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 302e213fd0cd4f1d8157d388bd82cf93, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732692331573 2024-11-27T07:25:33,301 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ff90d4d98d94d59a9141423f91b2e21, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732692331573 2024-11-27T07:25:33,326 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81ee2c488a54a955269ce99517372c4b#A#compaction#213 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:25:33,327 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/76970dca5ed34cb7b0f528be88fb378c is 50, key is test_row_0/A:col10/1732692332732/Put/seqid=0 2024-11-27T07:25:33,335 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81ee2c488a54a955269ce99517372c4b#B#compaction#214 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:25:33,335 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/a6550861808040889ade3e6059e5e781 is 50, key is test_row_0/B:col10/1732692332732/Put/seqid=0 2024-11-27T07:25:33,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742086_1262 (size=12241) 2024-11-27T07:25:33,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742087_1263 (size=12241) 2024-11-27T07:25:33,375 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/76970dca5ed34cb7b0f528be88fb378c as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/76970dca5ed34cb7b0f528be88fb378c 2024-11-27T07:25:33,381 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 81ee2c488a54a955269ce99517372c4b/A of 81ee2c488a54a955269ce99517372c4b into 76970dca5ed34cb7b0f528be88fb378c(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:25:33,382 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81ee2c488a54a955269ce99517372c4b: 2024-11-27T07:25:33,382 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., storeName=81ee2c488a54a955269ce99517372c4b/A, priority=12, startTime=1732692333286; duration=0sec 2024-11-27T07:25:33,382 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:25:33,382 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81ee2c488a54a955269ce99517372c4b:A 2024-11-27T07:25:33,382 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T07:25:33,384 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T07:25:33,384 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1541): 81ee2c488a54a955269ce99517372c4b/C is initiating minor compaction (all files) 2024-11-27T07:25:33,384 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81ee2c488a54a955269ce99517372c4b/C in TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:33,384 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/6416c66c71804937a58cca053720bfc2, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/002d440bf6a54d44854c3e4a3d5d6c37, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/c741face4aba407ca4f45e19870fcbbb, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/d8d757558f1f4b6f8eb6046b1ac3be87] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp, totalSize=47.0 K 2024-11-27T07:25:33,384 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6416c66c71804937a58cca053720bfc2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732692330187 2024-11-27T07:25:33,385 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 002d440bf6a54d44854c3e4a3d5d6c37, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732692330246 2024-11-27T07:25:33,385 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting c741face4aba407ca4f45e19870fcbbb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732692330900 2024-11-27T07:25:33,385 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8d757558f1f4b6f8eb6046b1ac3be87, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732692331573 2024-11-27T07:25:33,398 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81ee2c488a54a955269ce99517372c4b#C#compaction#215 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:25:33,399 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/39ea6c02dc1f4c0cbbc35c5b35c7fc41 is 50, key is test_row_0/C:col10/1732692332732/Put/seqid=0 2024-11-27T07:25:33,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742088_1264 (size=12241) 2024-11-27T07:25:33,437 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/39ea6c02dc1f4c0cbbc35c5b35c7fc41 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/39ea6c02dc1f4c0cbbc35c5b35c7fc41 2024-11-27T07:25:33,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 81ee2c488a54a955269ce99517372c4b 2024-11-27T07:25:33,442 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 81ee2c488a54a955269ce99517372c4b/C of 81ee2c488a54a955269ce99517372c4b into 39ea6c02dc1f4c0cbbc35c5b35c7fc41(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:25:33,442 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81ee2c488a54a955269ce99517372c4b: 2024-11-27T07:25:33,443 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T07:25:33,443 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., storeName=81ee2c488a54a955269ce99517372c4b/C, priority=12, startTime=1732692333287; duration=0sec 2024-11-27T07:25:33,443 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81ee2c488a54a955269ce99517372c4b 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-27T07:25:33,443 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:25:33,443 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81ee2c488a54a955269ce99517372c4b:C 2024-11-27T07:25:33,456 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/7f4791b0bba245dba3eeb559edca9c94 is 50, key is test_row_0/A:col10/1732692333441/Put/seqid=0 2024-11-27T07:25:33,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:33,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1732692393474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:33,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:33,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:33,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:33,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:33,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59268 deadline: 1732692393475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:33,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:33,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:33,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:33,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59340 deadline: 1732692393476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:33,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:33,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:33,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:33,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:33,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:33,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59326 deadline: 1732692393477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:33,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:33,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:33,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:33,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:33,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59334 deadline: 1732692393479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:33,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:33,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:33,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:33,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742089_1265 (size=12101) 2024-11-27T07:25:33,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:33,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1732692393589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:33,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:33,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:33,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:33,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:33,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59268 deadline: 1732692393589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:33,592 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:33,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:33,592 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:33,592 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:33,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59340 deadline: 1732692393589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:33,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:33,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:33,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:33,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:33,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59326 deadline: 1732692393589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:33,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:33,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:33,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59334 deadline: 1732692393589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:33,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:33,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:33,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:33,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:33,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:33,776 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/a6550861808040889ade3e6059e5e781 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/a6550861808040889ade3e6059e5e781 2024-11-27T07:25:33,784 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 81ee2c488a54a955269ce99517372c4b/B of 81ee2c488a54a955269ce99517372c4b into a6550861808040889ade3e6059e5e781(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:25:33,784 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81ee2c488a54a955269ce99517372c4b: 2024-11-27T07:25:33,784 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., storeName=81ee2c488a54a955269ce99517372c4b/B, priority=12, startTime=1732692333287; duration=0sec 2024-11-27T07:25:33,784 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:25:33,784 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81ee2c488a54a955269ce99517372c4b:B 2024-11-27T07:25:33,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:33,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59268 deadline: 1732692393799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:33,802 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:33,802 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:33,802 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:33,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:33,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59340 deadline: 1732692393799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:33,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:33,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:33,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:33,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:33,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59326 deadline: 1732692393799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:33,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:33,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1732692393799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:33,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:33,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:33,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:33,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:33,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:33,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:33,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:33,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59334 deadline: 1732692393799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:33,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:33,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:33,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:33,887 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/7f4791b0bba245dba3eeb559edca9c94 2024-11-27T07:25:33,906 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/cc3dca164e004fceb603c972afbca92d is 50, key is test_row_0/B:col10/1732692333441/Put/seqid=0 2024-11-27T07:25:33,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742090_1266 (size=12101) 2024-11-27T07:25:33,927 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/cc3dca164e004fceb603c972afbca92d 2024-11-27T07:25:33,939 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/8742f5fd16df455c9d26d37aef80fb9c is 50, key is test_row_0/C:col10/1732692333441/Put/seqid=0 2024-11-27T07:25:33,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742091_1267 (size=12101) 2024-11-27T07:25:33,958 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/8742f5fd16df455c9d26d37aef80fb9c 2024-11-27T07:25:33,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/7f4791b0bba245dba3eeb559edca9c94 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/7f4791b0bba245dba3eeb559edca9c94 2024-11-27T07:25:33,978 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/7f4791b0bba245dba3eeb559edca9c94, entries=150, sequenceid=131, filesize=11.8 K 2024-11-27T07:25:33,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/cc3dca164e004fceb603c972afbca92d as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/cc3dca164e004fceb603c972afbca92d 2024-11-27T07:25:33,990 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/cc3dca164e004fceb603c972afbca92d, entries=150, sequenceid=131, filesize=11.8 K 2024-11-27T07:25:33,992 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/8742f5fd16df455c9d26d37aef80fb9c as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/8742f5fd16df455c9d26d37aef80fb9c 2024-11-27T07:25:34,003 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/8742f5fd16df455c9d26d37aef80fb9c, entries=150, sequenceid=131, filesize=11.8 K 2024-11-27T07:25:34,004 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 81ee2c488a54a955269ce99517372c4b in 561ms, sequenceid=131, compaction requested=false 2024-11-27T07:25:34,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81ee2c488a54a955269ce99517372c4b: 2024-11-27T07:25:34,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 81ee2c488a54a955269ce99517372c4b 2024-11-27T07:25:34,112 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T07:25:34,112 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81ee2c488a54a955269ce99517372c4b 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-27T07:25:34,124 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/24f26cc9e74f41e3ad23e44149be838f is 50, key is test_row_0/A:col10/1732692333471/Put/seqid=0 2024-11-27T07:25:34,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:34,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59340 deadline: 1732692394122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:34,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:34,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:34,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:34,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742092_1268 (size=12151) 2024-11-27T07:25:34,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:34,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59334 deadline: 1732692394124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:34,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:34,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:34,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:34,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:34,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59326 deadline: 1732692394126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:34,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:34,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:34,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:34,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:34,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59268 deadline: 1732692394126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:34,135 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:34,135 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:34,135 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:34,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:34,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1732692394133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:34,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:34,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:34,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:34,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:34,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59340 deadline: 1732692394229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:34,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:34,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:34,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:34,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:34,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59334 deadline: 1732692394239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:34,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:34,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:34,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:34,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59268 deadline: 1732692394239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:34,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:34,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:34,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:34,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:34,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:34,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59326 deadline: 1732692394239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:34,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:34,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:34,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:34,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:34,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1732692394240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:34,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:34,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:34,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:34,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-27T07:25:34,299 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T07:25:34,299 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T07:25:34,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-27T07:25:34,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-27T07:25:34,302 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T07:25:34,302 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T07:25:34,302 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T07:25:34,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-27T07:25:34,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:34,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59340 deadline: 1732692394439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:34,442 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:34,442 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:34,442 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:34,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:34,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59334 deadline: 1732692394449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:34,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:34,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:34,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59268 deadline: 1732692394449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:34,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:34,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:34,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:34,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:34,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:34,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:34,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59326 deadline: 1732692394450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:34,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:34,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:34,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:34,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:34,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1732692394450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:34,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:34,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:34,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:34,453 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-27T07:25:34,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:34,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. as already flushing 2024-11-27T07:25:34,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:34,453 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:34,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:34,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:34,529 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/24f26cc9e74f41e3ad23e44149be838f 2024-11-27T07:25:34,536 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/4fd6849935d343dc99efa8608dd74f46 is 50, key is test_row_0/B:col10/1732692333471/Put/seqid=0 2024-11-27T07:25:34,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742093_1269 (size=12151) 2024-11-27T07:25:34,606 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-27T07:25:34,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:34,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. as already flushing 2024-11-27T07:25:34,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:34,606 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:34,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:34,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:34,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-27T07:25:34,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:34,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59340 deadline: 1732692394749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:34,751 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:34,751 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:34,751 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:34,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-27T07:25:34,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:34,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. as already flushing 2024-11-27T07:25:34,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:34,760 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:34,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:34,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:34,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59334 deadline: 1732692394759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:34,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:34,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:34,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59268 deadline: 1732692394760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:34,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59326 deadline: 1732692394760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:34,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:34,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:34,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:34,761 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:34,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1732692394760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:34,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:34,761 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:34,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:34,761 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:34,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:34,762 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:34,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:34,762 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:34,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:34,762 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:34,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-27T07:25:34,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:34,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. as already flushing 2024-11-27T07:25:34,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:34,914 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:34,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:34,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:34,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-27T07:25:34,949 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/4fd6849935d343dc99efa8608dd74f46 2024-11-27T07:25:34,958 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/e54bd2f77db246a1816bf09c0fa74712 is 50, key is test_row_0/C:col10/1732692333471/Put/seqid=0 2024-11-27T07:25:34,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742094_1270 (size=12151) 2024-11-27T07:25:35,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-27T07:25:35,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:35,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. as already flushing 2024-11-27T07:25:35,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:35,067 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:35,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:35,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:35,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-27T07:25:35,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:35,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. as already flushing 2024-11-27T07:25:35,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:35,219 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:35,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:35,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:35,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:35,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59340 deadline: 1732692395259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:35,260 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:35,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:35,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:35,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:35,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59326 deadline: 1732692395269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:35,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:35,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:35,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:35,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:35,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59268 deadline: 1732692395269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:35,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:35,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:35,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59334 deadline: 1732692395270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:35,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:35,273 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:35,273 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:35,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1732692395270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:35,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:35,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:35,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:35,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:35,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:35,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:35,363 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/e54bd2f77db246a1816bf09c0fa74712 2024-11-27T07:25:35,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/24f26cc9e74f41e3ad23e44149be838f as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/24f26cc9e74f41e3ad23e44149be838f 2024-11-27T07:25:35,372 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/24f26cc9e74f41e3ad23e44149be838f, entries=150, sequenceid=158, filesize=11.9 K 2024-11-27T07:25:35,372 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-27T07:25:35,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:35,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. as already flushing 2024-11-27T07:25:35,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:35,372 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:35,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:35,373 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/4fd6849935d343dc99efa8608dd74f46 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/4fd6849935d343dc99efa8608dd74f46 2024-11-27T07:25:35,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:35,378 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/4fd6849935d343dc99efa8608dd74f46, entries=150, sequenceid=158, filesize=11.9 K 2024-11-27T07:25:35,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/e54bd2f77db246a1816bf09c0fa74712 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/e54bd2f77db246a1816bf09c0fa74712 2024-11-27T07:25:35,383 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/e54bd2f77db246a1816bf09c0fa74712, entries=150, sequenceid=158, filesize=11.9 K 2024-11-27T07:25:35,383 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for 81ee2c488a54a955269ce99517372c4b in 1271ms, sequenceid=158, compaction requested=true 2024-11-27T07:25:35,383 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81ee2c488a54a955269ce99517372c4b: 2024-11-27T07:25:35,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81ee2c488a54a955269ce99517372c4b:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T07:25:35,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:25:35,384 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:25:35,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81ee2c488a54a955269ce99517372c4b:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T07:25:35,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:25:35,384 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:25:35,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81ee2c488a54a955269ce99517372c4b:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T07:25:35,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:25:35,384 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36493 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:25:35,384 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36493 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:25:35,384 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1541): 81ee2c488a54a955269ce99517372c4b/A is initiating minor compaction (all files) 2024-11-27T07:25:35,384 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 81ee2c488a54a955269ce99517372c4b/B is initiating minor compaction (all files) 2024-11-27T07:25:35,385 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81ee2c488a54a955269ce99517372c4b/B in TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:35,385 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81ee2c488a54a955269ce99517372c4b/A in TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:35,385 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/a6550861808040889ade3e6059e5e781, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/cc3dca164e004fceb603c972afbca92d, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/4fd6849935d343dc99efa8608dd74f46] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp, totalSize=35.6 K 2024-11-27T07:25:35,385 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/76970dca5ed34cb7b0f528be88fb378c, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/7f4791b0bba245dba3eeb559edca9c94, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/24f26cc9e74f41e3ad23e44149be838f] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp, totalSize=35.6 K 2024-11-27T07:25:35,386 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76970dca5ed34cb7b0f528be88fb378c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732692331573 2024-11-27T07:25:35,386 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting a6550861808040889ade3e6059e5e781, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732692331573 2024-11-27T07:25:35,386 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f4791b0bba245dba3eeb559edca9c94, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732692332758 2024-11-27T07:25:35,386 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting cc3dca164e004fceb603c972afbca92d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732692332758 2024-11-27T07:25:35,386 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24f26cc9e74f41e3ad23e44149be838f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732692333471 2024-11-27T07:25:35,386 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fd6849935d343dc99efa8608dd74f46, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732692333471 2024-11-27T07:25:35,393 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81ee2c488a54a955269ce99517372c4b#A#compaction#223 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:25:35,394 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/2013c82fabcd4ecb924b80aaae329026 is 50, key is test_row_0/A:col10/1732692333471/Put/seqid=0 2024-11-27T07:25:35,395 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81ee2c488a54a955269ce99517372c4b#B#compaction#222 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:25:35,396 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/89e0f68310814734967444b2eb3a912d is 50, key is test_row_0/B:col10/1732692333471/Put/seqid=0 2024-11-27T07:25:35,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742096_1272 (size=12493) 2024-11-27T07:25:35,421 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/89e0f68310814734967444b2eb3a912d as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/89e0f68310814734967444b2eb3a912d 2024-11-27T07:25:35,428 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81ee2c488a54a955269ce99517372c4b/B of 81ee2c488a54a955269ce99517372c4b into 89e0f68310814734967444b2eb3a912d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:25:35,428 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81ee2c488a54a955269ce99517372c4b: 2024-11-27T07:25:35,428 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., storeName=81ee2c488a54a955269ce99517372c4b/B, priority=13, startTime=1732692335384; duration=0sec 2024-11-27T07:25:35,428 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:25:35,428 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81ee2c488a54a955269ce99517372c4b:B 2024-11-27T07:25:35,428 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:25:35,430 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36493 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:25:35,430 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 81ee2c488a54a955269ce99517372c4b/C is initiating minor compaction (all files) 2024-11-27T07:25:35,430 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81ee2c488a54a955269ce99517372c4b/C in TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:35,431 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/39ea6c02dc1f4c0cbbc35c5b35c7fc41, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/8742f5fd16df455c9d26d37aef80fb9c, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/e54bd2f77db246a1816bf09c0fa74712] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp, totalSize=35.6 K 2024-11-27T07:25:35,431 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 39ea6c02dc1f4c0cbbc35c5b35c7fc41, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732692331573 2024-11-27T07:25:35,431 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 8742f5fd16df455c9d26d37aef80fb9c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732692332758 2024-11-27T07:25:35,432 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting e54bd2f77db246a1816bf09c0fa74712, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732692333471 2024-11-27T07:25:35,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-27T07:25:35,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742095_1271 (size=12493) 2024-11-27T07:25:35,458 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/2013c82fabcd4ecb924b80aaae329026 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/2013c82fabcd4ecb924b80aaae329026 2024-11-27T07:25:35,462 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81ee2c488a54a955269ce99517372c4b#C#compaction#224 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:25:35,462 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/ae9744423683402ab0282a7169800cd0 is 50, key is test_row_0/C:col10/1732692333471/Put/seqid=0 2024-11-27T07:25:35,464 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81ee2c488a54a955269ce99517372c4b/A of 81ee2c488a54a955269ce99517372c4b into 2013c82fabcd4ecb924b80aaae329026(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:25:35,465 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81ee2c488a54a955269ce99517372c4b: 2024-11-27T07:25:35,465 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., storeName=81ee2c488a54a955269ce99517372c4b/A, priority=13, startTime=1732692335383; duration=0sec 2024-11-27T07:25:35,465 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:25:35,466 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81ee2c488a54a955269ce99517372c4b:A 2024-11-27T07:25:35,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742097_1273 (size=12493) 2024-11-27T07:25:35,517 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/ae9744423683402ab0282a7169800cd0 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/ae9744423683402ab0282a7169800cd0 2024-11-27T07:25:35,524 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81ee2c488a54a955269ce99517372c4b/C of 81ee2c488a54a955269ce99517372c4b into ae9744423683402ab0282a7169800cd0(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:25:35,524 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81ee2c488a54a955269ce99517372c4b: 2024-11-27T07:25:35,524 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., storeName=81ee2c488a54a955269ce99517372c4b/C, priority=13, startTime=1732692335384; duration=0sec 2024-11-27T07:25:35,524 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:25:35,524 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81ee2c488a54a955269ce99517372c4b:C 2024-11-27T07:25:35,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-27T07:25:35,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:35,525 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 81ee2c488a54a955269ce99517372c4b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-27T07:25:35,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/0396eacaa36647528c26c535aa147621 is 50, key is test_row_0/A:col10/1732692334124/Put/seqid=0 2024-11-27T07:25:35,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742098_1274 (size=12151) 2024-11-27T07:25:35,574 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/0396eacaa36647528c26c535aa147621 2024-11-27T07:25:35,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/f7b527acf1ba46dca80aa4a9ad4b3c98 is 50, key is test_row_0/B:col10/1732692334124/Put/seqid=0 2024-11-27T07:25:35,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742099_1275 (size=12151) 2024-11-27T07:25:35,636 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/f7b527acf1ba46dca80aa4a9ad4b3c98 2024-11-27T07:25:35,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/d897fca4019a4e4290678762bee4b8be is 50, key is test_row_0/C:col10/1732692334124/Put/seqid=0 2024-11-27T07:25:35,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742100_1276 (size=12151) 2024-11-27T07:25:35,673 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/d897fca4019a4e4290678762bee4b8be 2024-11-27T07:25:35,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/0396eacaa36647528c26c535aa147621 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/0396eacaa36647528c26c535aa147621 2024-11-27T07:25:35,692 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/0396eacaa36647528c26c535aa147621, entries=150, sequenceid=172, filesize=11.9 K 2024-11-27T07:25:35,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/f7b527acf1ba46dca80aa4a9ad4b3c98 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/f7b527acf1ba46dca80aa4a9ad4b3c98 2024-11-27T07:25:35,718 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/f7b527acf1ba46dca80aa4a9ad4b3c98, entries=150, sequenceid=172, filesize=11.9 K 2024-11-27T07:25:35,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/d897fca4019a4e4290678762bee4b8be as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/d897fca4019a4e4290678762bee4b8be 2024-11-27T07:25:35,726 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/d897fca4019a4e4290678762bee4b8be, entries=150, sequenceid=172, filesize=11.9 K 2024-11-27T07:25:35,728 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=0 B/0 for 81ee2c488a54a955269ce99517372c4b in 203ms, sequenceid=172, compaction requested=false 2024-11-27T07:25:35,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 81ee2c488a54a955269ce99517372c4b: 2024-11-27T07:25:35,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:35,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-27T07:25:35,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-27T07:25:35,733 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-27T07:25:35,733 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4280 sec 2024-11-27T07:25:35,735 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.4340 sec 2024-11-27T07:25:36,286 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T07:25:36,287 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81ee2c488a54a955269ce99517372c4b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-27T07:25:36,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 81ee2c488a54a955269ce99517372c4b 2024-11-27T07:25:36,297 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/478011dcba0549749a47d070e4b401c0 is 50, key is test_row_0/A:col10/1732692336285/Put/seqid=0 2024-11-27T07:25:36,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742101_1277 (size=12151) 2024-11-27T07:25:36,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:36,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:36,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59326 deadline: 1732692396319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:36,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59334 deadline: 1732692396322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:36,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:36,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:36,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:36,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:36,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:36,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:36,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:36,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1732692396363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:36,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:36,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:36,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:36,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59268 deadline: 1732692396363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:36,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:36,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:36,368 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:36,368 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:36,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59340 deadline: 1732692396363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:36,368 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:36,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:36,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:36,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:36,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-27T07:25:36,450 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T07:25:36,450 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T07:25:36,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-27T07:25:36,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-27T07:25:36,452 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T07:25:36,453 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T07:25:36,453 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T07:25:36,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:36,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59326 deadline: 1732692396469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:36,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:36,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:36,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:36,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:36,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59334 deadline: 1732692396469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:36,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:36,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:36,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:36,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:36,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1732692396479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:36,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:36,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:36,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:36,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:36,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59268 deadline: 1732692396479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:36,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:36,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:36,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:36,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:36,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59340 deadline: 1732692396479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:36,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:36,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:36,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:36,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-27T07:25:36,608 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-27T07:25:36,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:36,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. as already flushing 2024-11-27T07:25:36,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:36,608 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:36,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:36,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:36,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:36,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59326 deadline: 1732692396679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:36,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:36,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:36,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:36,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:36,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59334 deadline: 1732692396679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:36,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:36,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:36,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:36,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:36,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1732692396689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:36,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:36,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:36,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:36,691 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:36,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59268 deadline: 1732692396689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:36,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:36,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:36,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:36,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:36,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59340 deadline: 1732692396689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:36,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:36,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:36,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:36,719 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/478011dcba0549749a47d070e4b401c0 2024-11-27T07:25:36,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/1ac2e23b2c2949c19cb102671475ff35 is 50, key is test_row_0/B:col10/1732692336285/Put/seqid=0 2024-11-27T07:25:36,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742102_1278 (size=12151) 2024-11-27T07:25:36,761 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-27T07:25:36,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:36,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. as already flushing 2024-11-27T07:25:36,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:36,762 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:36,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:36,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:36,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-27T07:25:36,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-27T07:25:36,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:36,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. as already flushing 2024-11-27T07:25:36,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:36,914 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:36,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:36,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:36,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:36,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59326 deadline: 1732692396992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:36,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:36,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:36,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:36,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:36,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59334 deadline: 1732692396993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:36,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:36,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:36,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:37,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:37,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1732692396999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:37,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:37,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59340 deadline: 1732692397000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:37,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:37,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59268 deadline: 1732692397000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:37,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:37,004 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:37,004 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:37,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:37,004 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:37,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:37,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:37,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:37,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:37,067 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-27T07:25:37,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:37,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. as already flushing 2024-11-27T07:25:37,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:37,067 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:37,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:37,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:37,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-27T07:25:37,129 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/1ac2e23b2c2949c19cb102671475ff35 2024-11-27T07:25:37,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/5a9c539e17464a048705ac7a0c4b90b3 is 50, key is test_row_0/C:col10/1732692336285/Put/seqid=0 2024-11-27T07:25:37,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742103_1279 (size=12151) 2024-11-27T07:25:37,220 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-27T07:25:37,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:37,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. as already flushing 2024-11-27T07:25:37,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:37,221 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:37,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:37,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:37,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-27T07:25:37,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:37,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. as already flushing 2024-11-27T07:25:37,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:37,374 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:37,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:37,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:37,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:37,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59326 deadline: 1732692397499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:37,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:37,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:37,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:37,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:37,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59334 deadline: 1732692397509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:37,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:37,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1732692397509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:37,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:37,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:37,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:37,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:37,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59340 deadline: 1732692397510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:37,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:37,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:37,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:37,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:37,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59268 deadline: 1732692397510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:37,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:37,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:37,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:37,513 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:37,513 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:37,513 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:37,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-27T07:25:37,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:37,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. as already flushing 2024-11-27T07:25:37,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:37,528 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:37,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:37,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:37,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-27T07:25:37,592 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/5a9c539e17464a048705ac7a0c4b90b3 2024-11-27T07:25:37,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/478011dcba0549749a47d070e4b401c0 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/478011dcba0549749a47d070e4b401c0 2024-11-27T07:25:37,604 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/478011dcba0549749a47d070e4b401c0, entries=150, sequenceid=183, filesize=11.9 K 2024-11-27T07:25:37,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/1ac2e23b2c2949c19cb102671475ff35 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/1ac2e23b2c2949c19cb102671475ff35 2024-11-27T07:25:37,614 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/1ac2e23b2c2949c19cb102671475ff35, entries=150, sequenceid=183, filesize=11.9 K 2024-11-27T07:25:37,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/5a9c539e17464a048705ac7a0c4b90b3 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/5a9c539e17464a048705ac7a0c4b90b3 2024-11-27T07:25:37,624 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/5a9c539e17464a048705ac7a0c4b90b3, entries=150, sequenceid=183, filesize=11.9 K 2024-11-27T07:25:37,624 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 81ee2c488a54a955269ce99517372c4b in 1338ms, sequenceid=183, compaction requested=true 2024-11-27T07:25:37,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81ee2c488a54a955269ce99517372c4b: 2024-11-27T07:25:37,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81ee2c488a54a955269ce99517372c4b:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T07:25:37,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:25:37,625 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:25:37,625 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:25:37,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81ee2c488a54a955269ce99517372c4b:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T07:25:37,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:25:37,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81ee2c488a54a955269ce99517372c4b:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T07:25:37,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:25:37,626 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:25:37,626 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 81ee2c488a54a955269ce99517372c4b/B is initiating minor compaction (all files) 2024-11-27T07:25:37,626 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81ee2c488a54a955269ce99517372c4b/B in TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:37,626 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/89e0f68310814734967444b2eb3a912d, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/f7b527acf1ba46dca80aa4a9ad4b3c98, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/1ac2e23b2c2949c19cb102671475ff35] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp, totalSize=35.9 K 2024-11-27T07:25:37,626 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:25:37,626 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1541): 81ee2c488a54a955269ce99517372c4b/A is initiating minor compaction (all files) 2024-11-27T07:25:37,626 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81ee2c488a54a955269ce99517372c4b/A in TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:37,626 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/2013c82fabcd4ecb924b80aaae329026, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/0396eacaa36647528c26c535aa147621, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/478011dcba0549749a47d070e4b401c0] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp, totalSize=35.9 K 2024-11-27T07:25:37,627 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2013c82fabcd4ecb924b80aaae329026, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732692333471 2024-11-27T07:25:37,627 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 89e0f68310814734967444b2eb3a912d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732692333471 2024-11-27T07:25:37,629 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0396eacaa36647528c26c535aa147621, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732692334117 2024-11-27T07:25:37,629 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting f7b527acf1ba46dca80aa4a9ad4b3c98, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732692334117 2024-11-27T07:25:37,629 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 478011dcba0549749a47d070e4b401c0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732692336270 2024-11-27T07:25:37,629 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ac2e23b2c2949c19cb102671475ff35, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732692336270 2024-11-27T07:25:37,646 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81ee2c488a54a955269ce99517372c4b#A#compaction#231 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:25:37,647 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/0bc1ecd03d6048c2a88b69377e677a38 is 50, key is test_row_0/A:col10/1732692336285/Put/seqid=0 2024-11-27T07:25:37,654 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81ee2c488a54a955269ce99517372c4b#B#compaction#232 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:25:37,655 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/b29952c5b06942bd9316b1568c4dd0ef is 50, key is test_row_0/B:col10/1732692336285/Put/seqid=0 2024-11-27T07:25:37,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742104_1280 (size=12595) 2024-11-27T07:25:37,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-27T07:25:37,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:37,682 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 81ee2c488a54a955269ce99517372c4b 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-27T07:25:37,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742105_1281 (size=12595) 2024-11-27T07:25:37,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/3b5004f7da0e42faaad675fde7d6f454 is 50, key is test_row_0/A:col10/1732692336316/Put/seqid=0 2024-11-27T07:25:37,692 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/b29952c5b06942bd9316b1568c4dd0ef as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/b29952c5b06942bd9316b1568c4dd0ef 2024-11-27T07:25:37,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742106_1282 (size=12151) 2024-11-27T07:25:37,698 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81ee2c488a54a955269ce99517372c4b/B of 81ee2c488a54a955269ce99517372c4b into b29952c5b06942bd9316b1568c4dd0ef(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:25:37,698 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81ee2c488a54a955269ce99517372c4b: 2024-11-27T07:25:37,698 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., storeName=81ee2c488a54a955269ce99517372c4b/B, priority=13, startTime=1732692337625; duration=0sec 2024-11-27T07:25:37,698 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:25:37,699 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81ee2c488a54a955269ce99517372c4b:B 2024-11-27T07:25:37,699 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:25:37,703 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:25:37,703 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 81ee2c488a54a955269ce99517372c4b/C is initiating minor compaction (all files) 2024-11-27T07:25:37,704 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81ee2c488a54a955269ce99517372c4b/C in TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:37,704 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/ae9744423683402ab0282a7169800cd0, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/d897fca4019a4e4290678762bee4b8be, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/5a9c539e17464a048705ac7a0c4b90b3] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp, totalSize=35.9 K 2024-11-27T07:25:37,704 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting ae9744423683402ab0282a7169800cd0, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732692333471 2024-11-27T07:25:37,706 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting d897fca4019a4e4290678762bee4b8be, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732692334117 2024-11-27T07:25:37,706 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a9c539e17464a048705ac7a0c4b90b3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732692336270 2024-11-27T07:25:37,714 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81ee2c488a54a955269ce99517372c4b#C#compaction#234 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:25:37,714 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/e409e0fa9099429cbf87fc385b1823fe is 50, key is test_row_0/C:col10/1732692336285/Put/seqid=0 2024-11-27T07:25:37,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742107_1283 (size=12595) 2024-11-27T07:25:38,085 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/0bc1ecd03d6048c2a88b69377e677a38 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/0bc1ecd03d6048c2a88b69377e677a38 2024-11-27T07:25:38,094 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81ee2c488a54a955269ce99517372c4b/A of 81ee2c488a54a955269ce99517372c4b into 0bc1ecd03d6048c2a88b69377e677a38(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:25:38,094 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81ee2c488a54a955269ce99517372c4b: 2024-11-27T07:25:38,094 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., storeName=81ee2c488a54a955269ce99517372c4b/A, priority=13, startTime=1732692337625; duration=0sec 2024-11-27T07:25:38,094 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:25:38,094 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81ee2c488a54a955269ce99517372c4b:A 2024-11-27T07:25:38,095 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/3b5004f7da0e42faaad675fde7d6f454 2024-11-27T07:25:38,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/50dec500678e4be48ce54a7d37f7ba69 is 50, key is test_row_0/B:col10/1732692336316/Put/seqid=0 2024-11-27T07:25:38,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742108_1284 (size=12151) 2024-11-27T07:25:38,140 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/e409e0fa9099429cbf87fc385b1823fe as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/e409e0fa9099429cbf87fc385b1823fe 2024-11-27T07:25:38,153 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81ee2c488a54a955269ce99517372c4b/C of 81ee2c488a54a955269ce99517372c4b into e409e0fa9099429cbf87fc385b1823fe(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:25:38,153 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81ee2c488a54a955269ce99517372c4b: 2024-11-27T07:25:38,153 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., storeName=81ee2c488a54a955269ce99517372c4b/C, priority=13, startTime=1732692337625; duration=0sec 2024-11-27T07:25:38,153 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:25:38,153 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81ee2c488a54a955269ce99517372c4b:C 2024-11-27T07:25:38,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 81ee2c488a54a955269ce99517372c4b 2024-11-27T07:25:38,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. as already flushing 2024-11-27T07:25:38,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:38,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59340 deadline: 1732692398529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:38,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:38,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59268 deadline: 1732692398530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:38,533 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 289954c9202d,42117,1732692259671 for 'test_row_2' in 81ee2c488a54a955269ce99517372c4b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T07:25:38,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:38,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:38,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:38,534 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 289954c9202d,42117,1732692259671 for 'test_row_0' in 81ee2c488a54a955269ce99517372c4b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T07:25:38,534 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:38,534 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:38,534 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:38,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:38,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59334 deadline: 1732692398531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:38,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:38,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1732692398532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:38,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:38,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:38,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:38,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:38,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59326 deadline: 1732692398534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:38,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:38,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:38,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:38,536 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/50dec500678e4be48ce54a7d37f7ba69 2024-11-27T07:25:38,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:38,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:38,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:38,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/d937a88257784bff948ac15a833fb0c4 is 50, key is test_row_0/C:col10/1732692336316/Put/seqid=0 2024-11-27T07:25:38,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742109_1285 (size=12151) 2024-11-27T07:25:38,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-27T07:25:38,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:38,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59334 deadline: 1732692398639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:38,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:38,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:38,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:38,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:38,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59326 deadline: 1732692398639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:38,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:38,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:38,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:38,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:38,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1732692398639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:38,643 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:38,643 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:38,643 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:38,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:38,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59334 deadline: 1732692398849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:38,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:38,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:38,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:38,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:38,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59326 deadline: 1732692398849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:38,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:38,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:38,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:38,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:38,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1732692398849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:38,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:38,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:38,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:38,963 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/d937a88257784bff948ac15a833fb0c4 2024-11-27T07:25:38,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/3b5004f7da0e42faaad675fde7d6f454 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/3b5004f7da0e42faaad675fde7d6f454 2024-11-27T07:25:38,970 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/3b5004f7da0e42faaad675fde7d6f454, entries=150, sequenceid=208, filesize=11.9 K 2024-11-27T07:25:38,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/50dec500678e4be48ce54a7d37f7ba69 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/50dec500678e4be48ce54a7d37f7ba69 2024-11-27T07:25:38,979 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/50dec500678e4be48ce54a7d37f7ba69, entries=150, sequenceid=208, filesize=11.9 K 2024-11-27T07:25:38,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/d937a88257784bff948ac15a833fb0c4 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/d937a88257784bff948ac15a833fb0c4 2024-11-27T07:25:38,985 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/d937a88257784bff948ac15a833fb0c4, entries=150, sequenceid=208, filesize=11.9 K 2024-11-27T07:25:38,987 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 81ee2c488a54a955269ce99517372c4b in 1305ms, sequenceid=208, compaction requested=false 2024-11-27T07:25:38,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 81ee2c488a54a955269ce99517372c4b: 2024-11-27T07:25:38,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:38,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-27T07:25:38,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-27T07:25:38,991 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-27T07:25:38,991 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5360 sec 2024-11-27T07:25:38,994 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 2.5420 sec 2024-11-27T07:25:39,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 81ee2c488a54a955269ce99517372c4b 2024-11-27T07:25:39,162 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T07:25:39,162 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81ee2c488a54a955269ce99517372c4b 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-27T07:25:39,167 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/1644044bf4d640e5a877d4ef07caecac is 50, key is test_row_0/A:col10/1732692338531/Put/seqid=0 2024-11-27T07:25:39,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742110_1286 (size=16931) 2024-11-27T07:25:39,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:39,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59326 deadline: 1732692399189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:39,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:39,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:39,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:39,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:39,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1732692399191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:39,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:39,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:39,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:39,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:39,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59334 deadline: 1732692399193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:39,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:39,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:39,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:39,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:39,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1732692399299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:39,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:39,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59326 deadline: 1732692399299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:39,301 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:39,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:39,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:39,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:39,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:39,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:39,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:39,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59334 deadline: 1732692399309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:39,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:39,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:39,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:39,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:39,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1732692399509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:39,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:39,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:39,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:39,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:39,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59326 deadline: 1732692399509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:39,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:39,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:39,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:39,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:39,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59334 deadline: 1732692399519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:39,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:39,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:39,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:39,577 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/1644044bf4d640e5a877d4ef07caecac 2024-11-27T07:25:39,586 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/a2f39c2d0fe648698f5b86c30ed47034 is 50, key is test_row_0/B:col10/1732692338531/Put/seqid=0 2024-11-27T07:25:39,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742111_1287 (size=12151) 2024-11-27T07:25:39,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:39,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59326 deadline: 1732692399819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:39,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:39,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:39,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:39,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:39,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1732692399829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:39,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:39,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:39,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:39,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:39,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59334 deadline: 1732692399829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:39,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:39,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:39,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:40,002 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/a2f39c2d0fe648698f5b86c30ed47034 2024-11-27T07:25:40,008 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/c09dd39ddbb84157a180f42e2d0b5d76 is 50, key is test_row_0/C:col10/1732692338531/Put/seqid=0 2024-11-27T07:25:40,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742112_1288 (size=12151) 2024-11-27T07:25:40,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:40,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59326 deadline: 1732692400329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:40,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:40,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:40,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:40,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:40,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1732692400338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:40,340 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:40,340 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:40,340 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:40,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:40,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59334 deadline: 1732692400339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:40,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:40,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:40,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:40,417 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/c09dd39ddbb84157a180f42e2d0b5d76 2024-11-27T07:25:40,423 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/1644044bf4d640e5a877d4ef07caecac as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/1644044bf4d640e5a877d4ef07caecac 2024-11-27T07:25:40,429 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/1644044bf4d640e5a877d4ef07caecac, entries=250, sequenceid=225, filesize=16.5 K 2024-11-27T07:25:40,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/a2f39c2d0fe648698f5b86c30ed47034 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/a2f39c2d0fe648698f5b86c30ed47034 2024-11-27T07:25:40,434 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/a2f39c2d0fe648698f5b86c30ed47034, entries=150, sequenceid=225, filesize=11.9 K 2024-11-27T07:25:40,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/c09dd39ddbb84157a180f42e2d0b5d76 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/c09dd39ddbb84157a180f42e2d0b5d76 2024-11-27T07:25:40,442 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/c09dd39ddbb84157a180f42e2d0b5d76, entries=150, sequenceid=225, filesize=11.9 K 2024-11-27T07:25:40,443 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 81ee2c488a54a955269ce99517372c4b in 1281ms, sequenceid=225, compaction requested=true 2024-11-27T07:25:40,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81ee2c488a54a955269ce99517372c4b: 2024-11-27T07:25:40,443 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:25:40,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81ee2c488a54a955269ce99517372c4b:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T07:25:40,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:25:40,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81ee2c488a54a955269ce99517372c4b:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T07:25:40,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:25:40,444 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:25:40,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81ee2c488a54a955269ce99517372c4b:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T07:25:40,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:25:40,445 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41677 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:25:40,445 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:25:40,445 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1541): 81ee2c488a54a955269ce99517372c4b/A is initiating minor compaction (all files) 2024-11-27T07:25:40,445 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 81ee2c488a54a955269ce99517372c4b/B is initiating minor compaction (all files) 2024-11-27T07:25:40,445 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81ee2c488a54a955269ce99517372c4b/A in TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:40,445 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81ee2c488a54a955269ce99517372c4b/B in TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:40,445 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/0bc1ecd03d6048c2a88b69377e677a38, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/3b5004f7da0e42faaad675fde7d6f454, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/1644044bf4d640e5a877d4ef07caecac] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp, totalSize=40.7 K 2024-11-27T07:25:40,446 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/b29952c5b06942bd9316b1568c4dd0ef, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/50dec500678e4be48ce54a7d37f7ba69, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/a2f39c2d0fe648698f5b86c30ed47034] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp, totalSize=36.0 K 2024-11-27T07:25:40,446 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bc1ecd03d6048c2a88b69377e677a38, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732692336270 2024-11-27T07:25:40,446 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting b29952c5b06942bd9316b1568c4dd0ef, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732692336270 2024-11-27T07:25:40,446 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b5004f7da0e42faaad675fde7d6f454, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732692336316 2024-11-27T07:25:40,446 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 50dec500678e4be48ce54a7d37f7ba69, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732692336316 2024-11-27T07:25:40,446 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1644044bf4d640e5a877d4ef07caecac, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732692338530 2024-11-27T07:25:40,446 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting a2f39c2d0fe648698f5b86c30ed47034, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732692338531 2024-11-27T07:25:40,459 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81ee2c488a54a955269ce99517372c4b#B#compaction#240 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:25:40,460 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/5344398c28e64dae813f295a40c1dfd4 is 50, key is test_row_0/B:col10/1732692338531/Put/seqid=0 2024-11-27T07:25:40,463 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81ee2c488a54a955269ce99517372c4b#A#compaction#241 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:25:40,464 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/48cbceaec0614b31a79b83854e843699 is 50, key is test_row_0/A:col10/1732692338531/Put/seqid=0 2024-11-27T07:25:40,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742114_1290 (size=12697) 2024-11-27T07:25:40,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742113_1289 (size=12697) 2024-11-27T07:25:40,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 81ee2c488a54a955269ce99517372c4b 2024-11-27T07:25:40,542 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T07:25:40,542 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81ee2c488a54a955269ce99517372c4b 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-27T07:25:40,546 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/17f24cf244b24f1abf80d1eca6d74c79 is 50, key is test_row_0/A:col10/1732692339189/Put/seqid=0 2024-11-27T07:25:40,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742115_1291 (size=12151) 2024-11-27T07:25:40,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:40,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59340 deadline: 1732692400564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:40,568 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:40,568 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:40,568 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:40,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:40,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59268 deadline: 1732692400567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:40,571 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:40,571 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:40,571 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:40,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-27T07:25:40,619 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T07:25:40,620 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T07:25:40,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-27T07:25:40,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-27T07:25:40,622 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T07:25:40,622 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T07:25:40,622 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T07:25:40,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:40,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59340 deadline: 1732692400678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:40,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:40,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:40,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:40,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:40,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59268 deadline: 1732692400679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:40,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:40,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:40,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:40,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-27T07:25:40,774 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-27T07:25:40,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:40,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. as already flushing 2024-11-27T07:25:40,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:40,775 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:40,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:40,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:40,887 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/48cbceaec0614b31a79b83854e843699 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/48cbceaec0614b31a79b83854e843699 2024-11-27T07:25:40,892 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81ee2c488a54a955269ce99517372c4b/A of 81ee2c488a54a955269ce99517372c4b into 48cbceaec0614b31a79b83854e843699(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:25:40,892 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81ee2c488a54a955269ce99517372c4b: 2024-11-27T07:25:40,892 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., storeName=81ee2c488a54a955269ce99517372c4b/A, priority=13, startTime=1732692340443; duration=0sec 2024-11-27T07:25:40,892 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:25:40,892 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81ee2c488a54a955269ce99517372c4b:A 2024-11-27T07:25:40,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:40,892 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:25:40,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59268 deadline: 1732692400889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:40,892 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:40,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:40,892 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:40,892 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:40,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59340 deadline: 1732692400892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:40,893 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:40,893 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:40,893 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:40,893 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:25:40,893 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1541): 81ee2c488a54a955269ce99517372c4b/C is initiating minor compaction (all files) 2024-11-27T07:25:40,893 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81ee2c488a54a955269ce99517372c4b/C in TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:40,893 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/e409e0fa9099429cbf87fc385b1823fe, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/d937a88257784bff948ac15a833fb0c4, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/c09dd39ddbb84157a180f42e2d0b5d76] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp, totalSize=36.0 K 2024-11-27T07:25:40,894 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting e409e0fa9099429cbf87fc385b1823fe, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732692336270 2024-11-27T07:25:40,894 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting d937a88257784bff948ac15a833fb0c4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732692336316 2024-11-27T07:25:40,894 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting c09dd39ddbb84157a180f42e2d0b5d76, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732692338531 2024-11-27T07:25:40,906 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81ee2c488a54a955269ce99517372c4b#C#compaction#243 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:25:40,908 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/5344398c28e64dae813f295a40c1dfd4 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/5344398c28e64dae813f295a40c1dfd4 2024-11-27T07:25:40,909 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/0979171387b34c95922f844d6aa5faac is 50, key is test_row_0/C:col10/1732692338531/Put/seqid=0 2024-11-27T07:25:40,915 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81ee2c488a54a955269ce99517372c4b/B of 81ee2c488a54a955269ce99517372c4b into 5344398c28e64dae813f295a40c1dfd4(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:25:40,915 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81ee2c488a54a955269ce99517372c4b: 2024-11-27T07:25:40,915 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., storeName=81ee2c488a54a955269ce99517372c4b/B, priority=13, startTime=1732692340443; duration=0sec 2024-11-27T07:25:40,915 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:25:40,915 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81ee2c488a54a955269ce99517372c4b:B 2024-11-27T07:25:40,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742116_1292 (size=12697) 2024-11-27T07:25:40,927 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-27T07:25:40,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:40,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. as already flushing 2024-11-27T07:25:40,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:40,927 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:40,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:40,928 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/0979171387b34c95922f844d6aa5faac as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/0979171387b34c95922f844d6aa5faac 2024-11-27T07:25:40,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:40,933 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81ee2c488a54a955269ce99517372c4b/C of 81ee2c488a54a955269ce99517372c4b into 0979171387b34c95922f844d6aa5faac(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:25:40,933 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81ee2c488a54a955269ce99517372c4b: 2024-11-27T07:25:40,933 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., storeName=81ee2c488a54a955269ce99517372c4b/C, priority=13, startTime=1732692340444; duration=0sec 2024-11-27T07:25:40,933 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:25:40,933 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81ee2c488a54a955269ce99517372c4b:C 2024-11-27T07:25:40,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-27T07:25:40,951 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/17f24cf244b24f1abf80d1eca6d74c79 2024-11-27T07:25:40,959 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/c7c19c39f64b4a578acdb97504b62ba2 is 50, key is test_row_0/B:col10/1732692339189/Put/seqid=0 2024-11-27T07:25:40,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742117_1293 (size=12151) 2024-11-27T07:25:41,080 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-27T07:25:41,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:41,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. as already flushing 2024-11-27T07:25:41,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:41,080 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:41,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:41,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:41,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:41,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59268 deadline: 1732692401198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:41,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:41,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:41,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:41,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59340 deadline: 1732692401198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:41,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:41,201 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:41,201 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:41,201 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:41,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-27T07:25:41,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:41,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. as already flushing 2024-11-27T07:25:41,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:41,233 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:41,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:41,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:41,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-27T07:25:41,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:41,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59326 deadline: 1732692401339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:41,342 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 289954c9202d,42117,1732692259671 for 'test_row_0' in 81ee2c488a54a955269ce99517372c4b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T07:25:41,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:41,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:41,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:41,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:41,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59334 deadline: 1732692401349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:41,351 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 289954c9202d,42117,1732692259671 for 'test_row_2' in 81ee2c488a54a955269ce99517372c4b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T07:25:41,351 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:41,351 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:41,351 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:41,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:41,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1732692401359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:41,362 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 289954c9202d,42117,1732692259671 for 'test_row_1' in 81ee2c488a54a955269ce99517372c4b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T07:25:41,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:41,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:41,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:41,363 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/c7c19c39f64b4a578acdb97504b62ba2 2024-11-27T07:25:41,370 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/b3bcbef2226f4e7e86c17756f3640cd5 is 50, key is test_row_0/C:col10/1732692339189/Put/seqid=0 2024-11-27T07:25:41,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742118_1294 (size=12151) 2024-11-27T07:25:41,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-27T07:25:41,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:41,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. as already flushing 2024-11-27T07:25:41,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:41,386 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:41,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:41,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:41,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-27T07:25:41,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:41,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. as already flushing 2024-11-27T07:25:41,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:41,538 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:41,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:41,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:41,691 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-27T07:25:41,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:41,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. as already flushing 2024-11-27T07:25:41,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:41,691 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:41,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:41,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:41,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:41,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59268 deadline: 1732692401709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:41,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:41,711 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:41,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59340 deadline: 1732692401709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:41,711 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:41,711 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:41,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:41,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:41,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:41,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-27T07:25:41,774 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/b3bcbef2226f4e7e86c17756f3640cd5 2024-11-27T07:25:41,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/17f24cf244b24f1abf80d1eca6d74c79 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/17f24cf244b24f1abf80d1eca6d74c79 2024-11-27T07:25:41,786 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/17f24cf244b24f1abf80d1eca6d74c79, entries=150, sequenceid=248, filesize=11.9 K 2024-11-27T07:25:41,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/c7c19c39f64b4a578acdb97504b62ba2 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/c7c19c39f64b4a578acdb97504b62ba2 2024-11-27T07:25:41,791 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/c7c19c39f64b4a578acdb97504b62ba2, entries=150, sequenceid=248, filesize=11.9 K 2024-11-27T07:25:41,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/b3bcbef2226f4e7e86c17756f3640cd5 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/b3bcbef2226f4e7e86c17756f3640cd5 2024-11-27T07:25:41,795 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/b3bcbef2226f4e7e86c17756f3640cd5, entries=150, sequenceid=248, filesize=11.9 K 2024-11-27T07:25:41,796 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 81ee2c488a54a955269ce99517372c4b in 1254ms, sequenceid=248, compaction requested=false 2024-11-27T07:25:41,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81ee2c488a54a955269ce99517372c4b: 2024-11-27T07:25:41,844 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-27T07:25:41,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:41,844 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 81ee2c488a54a955269ce99517372c4b 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-27T07:25:41,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/5e32336eff4a4c7fac3d0965de3dfbb0 is 50, key is test_row_0/A:col10/1732692340552/Put/seqid=0 2024-11-27T07:25:41,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742119_1295 (size=12251) 2024-11-27T07:25:41,852 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/5e32336eff4a4c7fac3d0965de3dfbb0 2024-11-27T07:25:41,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/8c0be216c09a475cba4763fb7326544f is 50, key is test_row_0/B:col10/1732692340552/Put/seqid=0 2024-11-27T07:25:41,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742120_1296 (size=12251) 2024-11-27T07:25:42,262 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/8c0be216c09a475cba4763fb7326544f 2024-11-27T07:25:42,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/81b2149867654ae7b95e50dceb0733db is 50, key is test_row_0/C:col10/1732692340552/Put/seqid=0 2024-11-27T07:25:42,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742121_1297 (size=12251) 2024-11-27T07:25:42,672 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/81b2149867654ae7b95e50dceb0733db 2024-11-27T07:25:42,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/5e32336eff4a4c7fac3d0965de3dfbb0 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/5e32336eff4a4c7fac3d0965de3dfbb0 2024-11-27T07:25:42,680 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/5e32336eff4a4c7fac3d0965de3dfbb0, entries=150, sequenceid=264, filesize=12.0 K 2024-11-27T07:25:42,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/8c0be216c09a475cba4763fb7326544f as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/8c0be216c09a475cba4763fb7326544f 2024-11-27T07:25:42,685 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/8c0be216c09a475cba4763fb7326544f, entries=150, sequenceid=264, filesize=12.0 K 2024-11-27T07:25:42,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/81b2149867654ae7b95e50dceb0733db as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/81b2149867654ae7b95e50dceb0733db 2024-11-27T07:25:42,697 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/81b2149867654ae7b95e50dceb0733db, entries=150, sequenceid=264, filesize=12.0 K 2024-11-27T07:25:42,698 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=0 B/0 for 81ee2c488a54a955269ce99517372c4b in 854ms, sequenceid=264, compaction requested=true 2024-11-27T07:25:42,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 81ee2c488a54a955269ce99517372c4b: 2024-11-27T07:25:42,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:42,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-27T07:25:42,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-27T07:25:42,702 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-27T07:25:42,702 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0780 sec 2024-11-27T07:25:42,704 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 2.0820 sec 2024-11-27T07:25:42,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 81ee2c488a54a955269ce99517372c4b 2024-11-27T07:25:42,735 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T07:25:42,735 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81ee2c488a54a955269ce99517372c4b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-27T07:25:42,740 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/2e6f0941557c4a6f8018b35acdc9d95f is 50, key is test_row_0/A:col10/1732692342734/Put/seqid=0 2024-11-27T07:25:42,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742122_1298 (size=14741) 2024-11-27T07:25:42,744 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/2e6f0941557c4a6f8018b35acdc9d95f 2024-11-27T07:25:42,751 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/acabf716411a48d98f97da92c16df777 is 50, key is test_row_0/B:col10/1732692342734/Put/seqid=0 2024-11-27T07:25:42,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742123_1299 (size=12301) 2024-11-27T07:25:42,758 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/acabf716411a48d98f97da92c16df777 2024-11-27T07:25:42,765 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/1e8103d1b02d480d820019c3d28d5c1b is 50, key is test_row_0/C:col10/1732692342734/Put/seqid=0 2024-11-27T07:25:42,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-27T07:25:42,771 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T07:25:42,771 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T07:25:42,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-27T07:25:42,773 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T07:25:42,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-27T07:25:42,773 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T07:25:42,773 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T07:25:42,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742124_1300 (size=12301) 2024-11-27T07:25:42,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:42,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59268 deadline: 1732692402811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:42,816 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:42,816 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:42,816 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:42,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:42,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59340 deadline: 1732692402816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:42,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:42,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:42,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:42,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-27T07:25:42,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:42,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59268 deadline: 1732692402918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:42,920 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:42,920 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:42,921 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:42,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-27T07:25:42,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:42,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. as already flushing 2024-11-27T07:25:42,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:42,926 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:42,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:42,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:42,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:42,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59340 deadline: 1732692402929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:42,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:42,932 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:42,932 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:43,079 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-27T07:25:43,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:43,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. as already flushing 2024-11-27T07:25:43,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:43,079 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:43,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:43,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:43,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-27T07:25:43,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:43,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59268 deadline: 1732692403129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:43,133 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:43,133 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:43,133 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:43,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:43,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59340 deadline: 1732692403138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:43,143 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:43,143 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:43,143 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:43,179 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/1e8103d1b02d480d820019c3d28d5c1b 2024-11-27T07:25:43,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/2e6f0941557c4a6f8018b35acdc9d95f as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/2e6f0941557c4a6f8018b35acdc9d95f 2024-11-27T07:25:43,186 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/2e6f0941557c4a6f8018b35acdc9d95f, entries=200, sequenceid=275, filesize=14.4 K 2024-11-27T07:25:43,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/acabf716411a48d98f97da92c16df777 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/acabf716411a48d98f97da92c16df777 2024-11-27T07:25:43,189 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/acabf716411a48d98f97da92c16df777, entries=150, sequenceid=275, filesize=12.0 K 2024-11-27T07:25:43,191 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/1e8103d1b02d480d820019c3d28d5c1b as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/1e8103d1b02d480d820019c3d28d5c1b 2024-11-27T07:25:43,194 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/1e8103d1b02d480d820019c3d28d5c1b, entries=150, sequenceid=275, filesize=12.0 K 2024-11-27T07:25:43,195 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 81ee2c488a54a955269ce99517372c4b in 460ms, sequenceid=275, compaction requested=true 2024-11-27T07:25:43,195 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81ee2c488a54a955269ce99517372c4b: 2024-11-27T07:25:43,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81ee2c488a54a955269ce99517372c4b:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T07:25:43,195 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T07:25:43,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:25:43,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81ee2c488a54a955269ce99517372c4b:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T07:25:43,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:25:43,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81ee2c488a54a955269ce99517372c4b:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T07:25:43,195 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T07:25:43,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:25:43,196 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51840 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T07:25:43,196 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49400 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T07:25:43,196 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 81ee2c488a54a955269ce99517372c4b/B is initiating minor compaction (all files) 2024-11-27T07:25:43,196 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1541): 81ee2c488a54a955269ce99517372c4b/A is initiating minor compaction (all files) 2024-11-27T07:25:43,196 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81ee2c488a54a955269ce99517372c4b/A in TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:43,196 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81ee2c488a54a955269ce99517372c4b/B in TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:43,196 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/48cbceaec0614b31a79b83854e843699, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/17f24cf244b24f1abf80d1eca6d74c79, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/5e32336eff4a4c7fac3d0965de3dfbb0, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/2e6f0941557c4a6f8018b35acdc9d95f] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp, totalSize=50.6 K 2024-11-27T07:25:43,196 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/5344398c28e64dae813f295a40c1dfd4, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/c7c19c39f64b4a578acdb97504b62ba2, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/8c0be216c09a475cba4763fb7326544f, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/acabf716411a48d98f97da92c16df777] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp, totalSize=48.2 K 2024-11-27T07:25:43,197 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48cbceaec0614b31a79b83854e843699, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732692338531 2024-11-27T07:25:43,197 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 5344398c28e64dae813f295a40c1dfd4, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732692338531 2024-11-27T07:25:43,197 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17f24cf244b24f1abf80d1eca6d74c79, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732692339189 2024-11-27T07:25:43,197 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting c7c19c39f64b4a578acdb97504b62ba2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732692339189 2024-11-27T07:25:43,197 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e32336eff4a4c7fac3d0965de3dfbb0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1732692340552 2024-11-27T07:25:43,197 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c0be216c09a475cba4763fb7326544f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1732692340552 2024-11-27T07:25:43,197 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e6f0941557c4a6f8018b35acdc9d95f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732692342720 2024-11-27T07:25:43,197 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting acabf716411a48d98f97da92c16df777, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732692342720 2024-11-27T07:25:43,219 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81ee2c488a54a955269ce99517372c4b#B#compaction#252 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:25:43,219 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81ee2c488a54a955269ce99517372c4b#A#compaction#253 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:25:43,219 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/77e53d92d7de4ee9ade3876e0a19ee06 is 50, key is test_row_0/B:col10/1732692342734/Put/seqid=0 2024-11-27T07:25:43,219 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/34c965517d784550a67310c096349b77 is 50, key is test_row_0/A:col10/1732692342734/Put/seqid=0 2024-11-27T07:25:43,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-27T07:25:43,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:43,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742125_1301 (size=12983) 2024-11-27T07:25:43,232 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 81ee2c488a54a955269ce99517372c4b 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-27T07:25:43,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742126_1302 (size=12983) 2024-11-27T07:25:43,241 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/34c965517d784550a67310c096349b77 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/34c965517d784550a67310c096349b77 2024-11-27T07:25:43,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/0edfc3f986264752adc767bc079a972f is 50, key is test_row_0/A:col10/1732692342814/Put/seqid=0 2024-11-27T07:25:43,249 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/77e53d92d7de4ee9ade3876e0a19ee06 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/77e53d92d7de4ee9ade3876e0a19ee06 2024-11-27T07:25:43,251 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 81ee2c488a54a955269ce99517372c4b/A of 81ee2c488a54a955269ce99517372c4b into 34c965517d784550a67310c096349b77(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:25:43,251 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81ee2c488a54a955269ce99517372c4b: 2024-11-27T07:25:43,251 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., storeName=81ee2c488a54a955269ce99517372c4b/A, priority=12, startTime=1732692343195; duration=0sec 2024-11-27T07:25:43,251 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:25:43,251 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81ee2c488a54a955269ce99517372c4b:A 2024-11-27T07:25:43,251 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T07:25:43,253 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49400 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T07:25:43,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742127_1303 (size=12301) 2024-11-27T07:25:43,253 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1541): 81ee2c488a54a955269ce99517372c4b/C is initiating minor compaction (all files) 2024-11-27T07:25:43,253 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81ee2c488a54a955269ce99517372c4b/C in TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:43,253 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/0979171387b34c95922f844d6aa5faac, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/b3bcbef2226f4e7e86c17756f3640cd5, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/81b2149867654ae7b95e50dceb0733db, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/1e8103d1b02d480d820019c3d28d5c1b] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp, totalSize=48.2 K 2024-11-27T07:25:43,254 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0979171387b34c95922f844d6aa5faac, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732692338531 2024-11-27T07:25:43,255 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3bcbef2226f4e7e86c17756f3640cd5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732692339189 2024-11-27T07:25:43,256 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81b2149867654ae7b95e50dceb0733db, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1732692340552 2024-11-27T07:25:43,256 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 81ee2c488a54a955269ce99517372c4b/B of 81ee2c488a54a955269ce99517372c4b into 77e53d92d7de4ee9ade3876e0a19ee06(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:25:43,256 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81ee2c488a54a955269ce99517372c4b: 2024-11-27T07:25:43,256 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., storeName=81ee2c488a54a955269ce99517372c4b/B, priority=12, startTime=1732692343195; duration=0sec 2024-11-27T07:25:43,256 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:25:43,256 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81ee2c488a54a955269ce99517372c4b:B 2024-11-27T07:25:43,256 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e8103d1b02d480d820019c3d28d5c1b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732692342720 2024-11-27T07:25:43,257 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/0edfc3f986264752adc767bc079a972f 2024-11-27T07:25:43,269 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81ee2c488a54a955269ce99517372c4b#C#compaction#255 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:25:43,270 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/89be84149fce4092855bd56f0f45c0a8 is 50, key is test_row_0/C:col10/1732692342734/Put/seqid=0 2024-11-27T07:25:43,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/c37ccb40a6944a40982d3c2f5dc29565 is 50, key is test_row_0/B:col10/1732692342814/Put/seqid=0 2024-11-27T07:25:43,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742128_1304 (size=12983) 2024-11-27T07:25:43,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742129_1305 (size=12301) 2024-11-27T07:25:43,309 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/c37ccb40a6944a40982d3c2f5dc29565 2024-11-27T07:25:43,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/11da7fd827f1402891820c1bf867dc62 is 50, key is test_row_0/C:col10/1732692342814/Put/seqid=0 2024-11-27T07:25:43,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742130_1306 (size=12301) 2024-11-27T07:25:43,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 81ee2c488a54a955269ce99517372c4b 2024-11-27T07:25:43,361 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. as already flushing 2024-11-27T07:25:43,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:43,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1732692403375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:43,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:43,379 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:43,379 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:43,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:43,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59326 deadline: 1732692403378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:43,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:43,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:43,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:43,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:43,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59334 deadline: 1732692403379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:43,381 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 289954c9202d,42117,1732692259671 for 'test_row_2' in 81ee2c488a54a955269ce99517372c4b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T07:25:43,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:43,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:43,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:43,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-27T07:25:43,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:43,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59268 deadline: 1732692403439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:43,442 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:43,442 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:43,442 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:43,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:43,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59340 deadline: 1732692403449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:43,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:43,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:43,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:43,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:43,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1732692403489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:43,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:43,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:43,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:43,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:43,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59326 deadline: 1732692403489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:43,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:43,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:43,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:43,696 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/89be84149fce4092855bd56f0f45c0a8 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/89be84149fce4092855bd56f0f45c0a8 2024-11-27T07:25:43,702 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 81ee2c488a54a955269ce99517372c4b/C of 81ee2c488a54a955269ce99517372c4b into 89be84149fce4092855bd56f0f45c0a8(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:25:43,702 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81ee2c488a54a955269ce99517372c4b: 2024-11-27T07:25:43,702 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., storeName=81ee2c488a54a955269ce99517372c4b/C, priority=12, startTime=1732692343195; duration=0sec 2024-11-27T07:25:43,702 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:25:43,702 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81ee2c488a54a955269ce99517372c4b:C 2024-11-27T07:25:43,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:43,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59326 deadline: 1732692403699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:43,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:43,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:43,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:43,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:43,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1732692403699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:43,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:43,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:43,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:43,722 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/11da7fd827f1402891820c1bf867dc62 2024-11-27T07:25:43,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/0edfc3f986264752adc767bc079a972f as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/0edfc3f986264752adc767bc079a972f 2024-11-27T07:25:43,731 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/0edfc3f986264752adc767bc079a972f, entries=150, sequenceid=300, filesize=12.0 K 2024-11-27T07:25:43,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/c37ccb40a6944a40982d3c2f5dc29565 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/c37ccb40a6944a40982d3c2f5dc29565 2024-11-27T07:25:43,736 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/c37ccb40a6944a40982d3c2f5dc29565, entries=150, sequenceid=300, filesize=12.0 K 2024-11-27T07:25:43,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/11da7fd827f1402891820c1bf867dc62 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/11da7fd827f1402891820c1bf867dc62 2024-11-27T07:25:43,741 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/11da7fd827f1402891820c1bf867dc62, entries=150, sequenceid=300, filesize=12.0 K 2024-11-27T07:25:43,742 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 81ee2c488a54a955269ce99517372c4b in 510ms, sequenceid=300, compaction requested=false 2024-11-27T07:25:43,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 81ee2c488a54a955269ce99517372c4b: 2024-11-27T07:25:43,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:43,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-27T07:25:43,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-27T07:25:43,745 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-27T07:25:43,745 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 971 msec 2024-11-27T07:25:43,747 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 975 msec 2024-11-27T07:25:43,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-27T07:25:43,910 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T07:25:43,910 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T07:25:43,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-27T07:25:43,911 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T07:25:43,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-27T07:25:43,912 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T07:25:43,912 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T07:25:43,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 81ee2c488a54a955269ce99517372c4b 2024-11-27T07:25:43,952 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T07:25:43,952 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81ee2c488a54a955269ce99517372c4b 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-27T07:25:43,955 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/750d672226644223959bfa26bfc7bb26 is 50, key is test_row_0/A:col10/1732692343951/Put/seqid=0 2024-11-27T07:25:43,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742131_1307 (size=14741) 2024-11-27T07:25:44,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:44,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59268 deadline: 1732692404001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:44,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:44,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59340 deadline: 1732692404003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:44,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:44,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:44,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:44,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:44,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:44,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:44,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:44,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59326 deadline: 1732692404008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:44,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:44,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:44,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:44,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:44,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1732692404008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:44,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:44,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:44,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:44,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-27T07:25:44,064 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-27T07:25:44,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:44,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. as already flushing 2024-11-27T07:25:44,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:44,064 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:44,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:44,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:44,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:44,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59340 deadline: 1732692404108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:44,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:44,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:44,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:44,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:44,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59268 deadline: 1732692404119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:44,121 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:44,121 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:44,121 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:44,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-27T07:25:44,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:44,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. as already flushing 2024-11-27T07:25:44,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:44,217 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:44,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:44,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:44,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-27T07:25:44,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:44,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59340 deadline: 1732692404319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:44,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:44,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:44,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:44,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:44,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59268 deadline: 1732692404329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:44,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:44,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:44,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:44,362 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/750d672226644223959bfa26bfc7bb26 2024-11-27T07:25:44,368 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/81fe68fe690d4ac191121a68c154ae8e is 50, key is test_row_0/B:col10/1732692343951/Put/seqid=0 2024-11-27T07:25:44,370 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-27T07:25:44,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:44,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. as already flushing 2024-11-27T07:25:44,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:44,370 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:44,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:44,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:44,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742132_1308 (size=12301) 2024-11-27T07:25:44,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:44,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:44,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59326 deadline: 1732692404519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:44,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1732692404519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:44,522 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-27T07:25:44,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:44,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:44,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:44,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:44,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:44,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:44,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:44,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. as already flushing 2024-11-27T07:25:44,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:44,523 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:44,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:44,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:44,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-27T07:25:44,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:44,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59340 deadline: 1732692404629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:44,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:44,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:44,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:44,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:44,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59268 deadline: 1732692404638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:44,642 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:44,642 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:44,642 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:44,675 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-27T07:25:44,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:44,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. as already flushing 2024-11-27T07:25:44,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:44,676 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:44,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:44,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:44,774 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/81fe68fe690d4ac191121a68c154ae8e 2024-11-27T07:25:44,780 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/0bbb11043f7d4abc8c71064d2f3fefc2 is 50, key is test_row_0/C:col10/1732692343951/Put/seqid=0 2024-11-27T07:25:44,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742133_1309 (size=12301) 2024-11-27T07:25:44,828 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-27T07:25:44,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:44,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. as already flushing 2024-11-27T07:25:44,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:44,828 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:44,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:44,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:44,981 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-27T07:25:44,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:44,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. as already flushing 2024-11-27T07:25:44,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:44,981 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:44,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:44,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:45,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-27T07:25:45,134 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-27T07:25:45,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:45,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. as already flushing 2024-11-27T07:25:45,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:45,134 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:45,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:45,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:45,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:45,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59340 deadline: 1732692405139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:45,141 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:45,141 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:45,141 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:45,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:45,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59268 deadline: 1732692405149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:45,151 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:45,151 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:45,151 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:45,184 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/0bbb11043f7d4abc8c71064d2f3fefc2 2024-11-27T07:25:45,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/750d672226644223959bfa26bfc7bb26 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/750d672226644223959bfa26bfc7bb26 2024-11-27T07:25:45,194 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/750d672226644223959bfa26bfc7bb26, entries=200, sequenceid=315, filesize=14.4 K 2024-11-27T07:25:45,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/81fe68fe690d4ac191121a68c154ae8e as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/81fe68fe690d4ac191121a68c154ae8e 2024-11-27T07:25:45,198 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/81fe68fe690d4ac191121a68c154ae8e, entries=150, sequenceid=315, filesize=12.0 K 2024-11-27T07:25:45,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/0bbb11043f7d4abc8c71064d2f3fefc2 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/0bbb11043f7d4abc8c71064d2f3fefc2 2024-11-27T07:25:45,202 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/0bbb11043f7d4abc8c71064d2f3fefc2, entries=150, sequenceid=315, filesize=12.0 K 2024-11-27T07:25:45,203 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 81ee2c488a54a955269ce99517372c4b in 1251ms, sequenceid=315, compaction requested=true 2024-11-27T07:25:45,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81ee2c488a54a955269ce99517372c4b: 2024-11-27T07:25:45,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81ee2c488a54a955269ce99517372c4b:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T07:25:45,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:25:45,203 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:25:45,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81ee2c488a54a955269ce99517372c4b:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T07:25:45,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:25:45,203 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:25:45,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81ee2c488a54a955269ce99517372c4b:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T07:25:45,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:25:45,204 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:25:45,204 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:25:45,204 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1541): 81ee2c488a54a955269ce99517372c4b/A is initiating minor compaction (all files) 2024-11-27T07:25:45,204 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 81ee2c488a54a955269ce99517372c4b/B is initiating minor compaction (all files) 2024-11-27T07:25:45,204 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81ee2c488a54a955269ce99517372c4b/A in TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:45,204 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81ee2c488a54a955269ce99517372c4b/B in TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:45,205 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/34c965517d784550a67310c096349b77, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/0edfc3f986264752adc767bc079a972f, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/750d672226644223959bfa26bfc7bb26] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp, totalSize=39.1 K 2024-11-27T07:25:45,205 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/77e53d92d7de4ee9ade3876e0a19ee06, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/c37ccb40a6944a40982d3c2f5dc29565, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/81fe68fe690d4ac191121a68c154ae8e] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp, totalSize=36.7 K 2024-11-27T07:25:45,205 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34c965517d784550a67310c096349b77, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732692342720 2024-11-27T07:25:45,205 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 77e53d92d7de4ee9ade3876e0a19ee06, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732692342720 2024-11-27T07:25:45,205 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0edfc3f986264752adc767bc079a972f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1732692342783 2024-11-27T07:25:45,205 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting c37ccb40a6944a40982d3c2f5dc29565, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1732692342783 2024-11-27T07:25:45,205 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 750d672226644223959bfa26bfc7bb26, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732692343373 2024-11-27T07:25:45,205 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 81fe68fe690d4ac191121a68c154ae8e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732692343373 2024-11-27T07:25:45,211 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81ee2c488a54a955269ce99517372c4b#B#compaction#261 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:25:45,211 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/f7297aff273b43488169d7d878358f84 is 50, key is test_row_0/B:col10/1732692343951/Put/seqid=0 2024-11-27T07:25:45,214 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81ee2c488a54a955269ce99517372c4b#A#compaction#262 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:25:45,214 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/f7489dd2edcd4ff99f7492ffe7098fa3 is 50, key is test_row_0/A:col10/1732692343951/Put/seqid=0 2024-11-27T07:25:45,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742134_1310 (size=13085) 2024-11-27T07:25:45,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742135_1311 (size=13085) 2024-11-27T07:25:45,286 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-27T07:25:45,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:45,287 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 81ee2c488a54a955269ce99517372c4b 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-27T07:25:45,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/501d81e8a5ba4e2fb4c7d9f687be5a20 is 50, key is test_row_0/A:col10/1732692344001/Put/seqid=0 2024-11-27T07:25:45,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742136_1312 (size=12301) 2024-11-27T07:25:45,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 81ee2c488a54a955269ce99517372c4b 2024-11-27T07:25:45,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. as already flushing 2024-11-27T07:25:45,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:45,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59326 deadline: 1732692405552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:45,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:45,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:45,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:45,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1732692405553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:45,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:45,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:45,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:45,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:45,623 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/f7297aff273b43488169d7d878358f84 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/f7297aff273b43488169d7d878358f84 2024-11-27T07:25:45,627 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/f7489dd2edcd4ff99f7492ffe7098fa3 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/f7489dd2edcd4ff99f7492ffe7098fa3 2024-11-27T07:25:45,630 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81ee2c488a54a955269ce99517372c4b/B of 81ee2c488a54a955269ce99517372c4b into f7297aff273b43488169d7d878358f84(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:25:45,630 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81ee2c488a54a955269ce99517372c4b: 2024-11-27T07:25:45,630 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., storeName=81ee2c488a54a955269ce99517372c4b/B, priority=13, startTime=1732692345203; duration=0sec 2024-11-27T07:25:45,630 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:25:45,630 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81ee2c488a54a955269ce99517372c4b:B 2024-11-27T07:25:45,630 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:25:45,633 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:25:45,633 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 81ee2c488a54a955269ce99517372c4b/C is initiating minor compaction (all files) 2024-11-27T07:25:45,633 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81ee2c488a54a955269ce99517372c4b/C in TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:45,633 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/89be84149fce4092855bd56f0f45c0a8, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/11da7fd827f1402891820c1bf867dc62, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/0bbb11043f7d4abc8c71064d2f3fefc2] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp, totalSize=36.7 K 2024-11-27T07:25:45,634 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 89be84149fce4092855bd56f0f45c0a8, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732692342720 2024-11-27T07:25:45,635 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 11da7fd827f1402891820c1bf867dc62, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1732692342783 2024-11-27T07:25:45,636 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 0bbb11043f7d4abc8c71064d2f3fefc2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732692343373 2024-11-27T07:25:45,636 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81ee2c488a54a955269ce99517372c4b/A of 81ee2c488a54a955269ce99517372c4b into f7489dd2edcd4ff99f7492ffe7098fa3(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:25:45,636 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81ee2c488a54a955269ce99517372c4b: 2024-11-27T07:25:45,636 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., storeName=81ee2c488a54a955269ce99517372c4b/A, priority=13, startTime=1732692345203; duration=0sec 2024-11-27T07:25:45,637 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:25:45,637 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81ee2c488a54a955269ce99517372c4b:A 2024-11-27T07:25:45,649 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81ee2c488a54a955269ce99517372c4b#C#compaction#264 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:25:45,650 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/0975e37d6c7c4074957506ebb6610ff6 is 50, key is test_row_0/C:col10/1732692343951/Put/seqid=0 2024-11-27T07:25:45,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742137_1313 (size=13085) 2024-11-27T07:25:45,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:45,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59326 deadline: 1732692405659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:45,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:45,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1732692405659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:45,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:45,662 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:45,662 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:45,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:45,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:45,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:45,695 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/501d81e8a5ba4e2fb4c7d9f687be5a20 2024-11-27T07:25:45,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/69ee67ff9cc1410291ee1dc4cd560e26 is 50, key is test_row_0/B:col10/1732692344001/Put/seqid=0 2024-11-27T07:25:45,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742138_1314 (size=12301) 2024-11-27T07:25:45,723 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/69ee67ff9cc1410291ee1dc4cd560e26 2024-11-27T07:25:45,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/186e81a15a994457a9b3bbb867a4d9ec is 50, key is test_row_0/C:col10/1732692344001/Put/seqid=0 2024-11-27T07:25:45,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742139_1315 (size=12301) 2024-11-27T07:25:45,740 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/186e81a15a994457a9b3bbb867a4d9ec 2024-11-27T07:25:45,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/501d81e8a5ba4e2fb4c7d9f687be5a20 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/501d81e8a5ba4e2fb4c7d9f687be5a20 2024-11-27T07:25:45,753 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/501d81e8a5ba4e2fb4c7d9f687be5a20, entries=150, sequenceid=339, filesize=12.0 K 2024-11-27T07:25:45,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/69ee67ff9cc1410291ee1dc4cd560e26 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/69ee67ff9cc1410291ee1dc4cd560e26 2024-11-27T07:25:45,758 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/69ee67ff9cc1410291ee1dc4cd560e26, entries=150, sequenceid=339, filesize=12.0 K 2024-11-27T07:25:45,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/186e81a15a994457a9b3bbb867a4d9ec as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/186e81a15a994457a9b3bbb867a4d9ec 2024-11-27T07:25:45,762 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/186e81a15a994457a9b3bbb867a4d9ec, entries=150, sequenceid=339, filesize=12.0 K 2024-11-27T07:25:45,763 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 81ee2c488a54a955269ce99517372c4b in 476ms, sequenceid=339, compaction requested=false 2024-11-27T07:25:45,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 81ee2c488a54a955269ce99517372c4b: 2024-11-27T07:25:45,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:45,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-27T07:25:45,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-27T07:25:45,766 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-27T07:25:45,766 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8520 sec 2024-11-27T07:25:45,767 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.8560 sec 2024-11-27T07:25:45,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 81ee2c488a54a955269ce99517372c4b 2024-11-27T07:25:45,875 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T07:25:45,875 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81ee2c488a54a955269ce99517372c4b 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-27T07:25:45,879 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/3edf4287eb8b48e4a075d3adb4a54eba is 50, key is test_row_0/A:col10/1732692345871/Put/seqid=0 2024-11-27T07:25:45,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742140_1316 (size=17181) 2024-11-27T07:25:45,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:45,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1732692405925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:45,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:45,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:45,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:45,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:45,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59326 deadline: 1732692405926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:45,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:45,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:45,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:46,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:46,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1732692406039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:46,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:46,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:46,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:46,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:46,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59326 deadline: 1732692406039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:46,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:46,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:46,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:46,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-27T07:25:46,059 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T07:25:46,060 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T07:25:46,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-27T07:25:46,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-27T07:25:46,062 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T07:25:46,062 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T07:25:46,062 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T07:25:46,063 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/0975e37d6c7c4074957506ebb6610ff6 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/0975e37d6c7c4074957506ebb6610ff6 2024-11-27T07:25:46,068 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81ee2c488a54a955269ce99517372c4b/C of 81ee2c488a54a955269ce99517372c4b into 0975e37d6c7c4074957506ebb6610ff6(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:25:46,068 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81ee2c488a54a955269ce99517372c4b: 2024-11-27T07:25:46,068 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., storeName=81ee2c488a54a955269ce99517372c4b/C, priority=13, startTime=1732692345203; duration=0sec 2024-11-27T07:25:46,068 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:25:46,068 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81ee2c488a54a955269ce99517372c4b:C 2024-11-27T07:25:46,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:46,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59340 deadline: 1732692406149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:46,152 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 289954c9202d,42117,1732692259671 for 'test_row_2' in 81ee2c488a54a955269ce99517372c4b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2149 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T07:25:46,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:46,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:46,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:46,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:46,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59268 deadline: 1732692406159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:46,162 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 289954c9202d,42117,1732692259671 for 'test_row_1' in 81ee2c488a54a955269ce99517372c4b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T07:25:46,163 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:46,163 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:46,163 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:46,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-27T07:25:46,215 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-27T07:25:46,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:46,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. as already flushing 2024-11-27T07:25:46,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:46,216 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:46,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:46,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:46,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:46,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1732692406249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:46,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:46,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:46,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:46,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:46,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59326 deadline: 1732692406249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:46,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:46,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:46,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:46,283 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/3edf4287eb8b48e4a075d3adb4a54eba 2024-11-27T07:25:46,290 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/6fed829faaa2424d92a4baf14e7e661c is 50, key is test_row_0/B:col10/1732692345871/Put/seqid=0 2024-11-27T07:25:46,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742141_1317 (size=12301) 2024-11-27T07:25:46,368 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-27T07:25:46,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:46,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. as already flushing 2024-11-27T07:25:46,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:46,368 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:46,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:46,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:46,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-27T07:25:46,521 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-27T07:25:46,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:46,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. as already flushing 2024-11-27T07:25:46,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:46,521 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:46,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:46,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:46,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:46,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:46,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1732692406559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:46,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59326 deadline: 1732692406559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:46,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:46,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:46,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:46,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:46,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:46,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:46,674 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-27T07:25:46,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:46,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. as already flushing 2024-11-27T07:25:46,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:46,675 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:46,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:46,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:46,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-27T07:25:46,698 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/6fed829faaa2424d92a4baf14e7e661c 2024-11-27T07:25:46,709 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/9c2a1646ad094cd38f61292545bc99d2 is 50, key is test_row_0/C:col10/1732692345871/Put/seqid=0 2024-11-27T07:25:46,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742142_1318 (size=12301) 2024-11-27T07:25:46,827 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-27T07:25:46,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:46,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. as already flushing 2024-11-27T07:25:46,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:46,828 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:46,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:46,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:46,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-27T07:25:46,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:46,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. as already flushing 2024-11-27T07:25:46,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:46,980 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:46,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:46,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:47,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:47,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59326 deadline: 1732692407069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:47,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:47,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:47,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:47,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:47,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1732692407069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:47,073 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:47,074 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:47,074 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:47,122 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/9c2a1646ad094cd38f61292545bc99d2 2024-11-27T07:25:47,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-27T07:25:47,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:47,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. as already flushing 2024-11-27T07:25:47,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:47,133 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:47,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:47,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:47,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/3edf4287eb8b48e4a075d3adb4a54eba as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/3edf4287eb8b48e4a075d3adb4a54eba 2024-11-27T07:25:47,139 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/3edf4287eb8b48e4a075d3adb4a54eba, entries=250, sequenceid=355, filesize=16.8 K 2024-11-27T07:25:47,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/6fed829faaa2424d92a4baf14e7e661c as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/6fed829faaa2424d92a4baf14e7e661c 2024-11-27T07:25:47,144 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/6fed829faaa2424d92a4baf14e7e661c, entries=150, sequenceid=355, filesize=12.0 K 2024-11-27T07:25:47,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/9c2a1646ad094cd38f61292545bc99d2 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/9c2a1646ad094cd38f61292545bc99d2 2024-11-27T07:25:47,148 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/9c2a1646ad094cd38f61292545bc99d2, entries=150, sequenceid=355, filesize=12.0 K 2024-11-27T07:25:47,149 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 81ee2c488a54a955269ce99517372c4b in 1274ms, sequenceid=355, compaction requested=true 2024-11-27T07:25:47,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81ee2c488a54a955269ce99517372c4b: 2024-11-27T07:25:47,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81ee2c488a54a955269ce99517372c4b:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T07:25:47,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:25:47,149 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:25:47,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81ee2c488a54a955269ce99517372c4b:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T07:25:47,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:25:47,149 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:25:47,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81ee2c488a54a955269ce99517372c4b:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T07:25:47,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:25:47,150 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:25:47,150 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 81ee2c488a54a955269ce99517372c4b/B is initiating minor compaction (all files) 2024-11-27T07:25:47,150 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81ee2c488a54a955269ce99517372c4b/B in TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:47,150 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/f7297aff273b43488169d7d878358f84, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/69ee67ff9cc1410291ee1dc4cd560e26, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/6fed829faaa2424d92a4baf14e7e661c] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp, totalSize=36.8 K 2024-11-27T07:25:47,150 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42567 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:25:47,150 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1541): 81ee2c488a54a955269ce99517372c4b/A is initiating minor compaction (all files) 2024-11-27T07:25:47,150 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81ee2c488a54a955269ce99517372c4b/A in TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:47,150 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/f7489dd2edcd4ff99f7492ffe7098fa3, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/501d81e8a5ba4e2fb4c7d9f687be5a20, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/3edf4287eb8b48e4a075d3adb4a54eba] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp, totalSize=41.6 K 2024-11-27T07:25:47,151 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting f7297aff273b43488169d7d878358f84, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732692343373 2024-11-27T07:25:47,151 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7489dd2edcd4ff99f7492ffe7098fa3, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732692343373 2024-11-27T07:25:47,151 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 69ee67ff9cc1410291ee1dc4cd560e26, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1732692343996 2024-11-27T07:25:47,160 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 501d81e8a5ba4e2fb4c7d9f687be5a20, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1732692343996 2024-11-27T07:25:47,160 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 6fed829faaa2424d92a4baf14e7e661c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1732692345552 2024-11-27T07:25:47,161 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3edf4287eb8b48e4a075d3adb4a54eba, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1732692345547 2024-11-27T07:25:47,170 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81ee2c488a54a955269ce99517372c4b#B#compaction#271 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:25:47,170 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81ee2c488a54a955269ce99517372c4b#A#compaction#270 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:25:47,171 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/16457640c497411eb3a8fd6fe0bc9b08 is 50, key is test_row_0/A:col10/1732692345871/Put/seqid=0 2024-11-27T07:25:47,171 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/062aa03c3609481c8b61d8778f6af210 is 50, key is test_row_0/B:col10/1732692345871/Put/seqid=0 2024-11-27T07:25:47,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742143_1319 (size=13187) 2024-11-27T07:25:47,186 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/062aa03c3609481c8b61d8778f6af210 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/062aa03c3609481c8b61d8778f6af210 2024-11-27T07:25:47,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742144_1320 (size=13187) 2024-11-27T07:25:47,190 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81ee2c488a54a955269ce99517372c4b/B of 81ee2c488a54a955269ce99517372c4b into 062aa03c3609481c8b61d8778f6af210(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:25:47,190 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81ee2c488a54a955269ce99517372c4b: 2024-11-27T07:25:47,190 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., storeName=81ee2c488a54a955269ce99517372c4b/B, priority=13, startTime=1732692347149; duration=0sec 2024-11-27T07:25:47,190 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:25:47,190 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81ee2c488a54a955269ce99517372c4b:B 2024-11-27T07:25:47,190 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:25:47,191 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:25:47,191 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 81ee2c488a54a955269ce99517372c4b/C is initiating minor compaction (all files) 2024-11-27T07:25:47,191 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81ee2c488a54a955269ce99517372c4b/C in TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:47,191 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/0975e37d6c7c4074957506ebb6610ff6, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/186e81a15a994457a9b3bbb867a4d9ec, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/9c2a1646ad094cd38f61292545bc99d2] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp, totalSize=36.8 K 2024-11-27T07:25:47,192 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 0975e37d6c7c4074957506ebb6610ff6, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732692343373 2024-11-27T07:25:47,192 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 186e81a15a994457a9b3bbb867a4d9ec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1732692343996 2024-11-27T07:25:47,192 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c2a1646ad094cd38f61292545bc99d2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1732692345552 2024-11-27T07:25:47,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-27T07:25:47,200 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81ee2c488a54a955269ce99517372c4b#C#compaction#272 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:25:47,200 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/3c9e545c249b4833bc61ff5c999c9b36 is 50, key is test_row_0/C:col10/1732692345871/Put/seqid=0 2024-11-27T07:25:47,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742145_1321 (size=13187) 2024-11-27T07:25:47,231 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/3c9e545c249b4833bc61ff5c999c9b36 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/3c9e545c249b4833bc61ff5c999c9b36 2024-11-27T07:25:47,255 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81ee2c488a54a955269ce99517372c4b/C of 81ee2c488a54a955269ce99517372c4b into 3c9e545c249b4833bc61ff5c999c9b36(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:25:47,255 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81ee2c488a54a955269ce99517372c4b: 2024-11-27T07:25:47,255 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., storeName=81ee2c488a54a955269ce99517372c4b/C, priority=13, startTime=1732692347149; duration=0sec 2024-11-27T07:25:47,255 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:25:47,255 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81ee2c488a54a955269ce99517372c4b:C 2024-11-27T07:25:47,287 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-27T07:25:47,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:47,287 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing 81ee2c488a54a955269ce99517372c4b 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-27T07:25:47,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/4c3784c82125428680e3a066b981b575 is 50, key is test_row_0/A:col10/1732692345924/Put/seqid=0 2024-11-27T07:25:47,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742146_1322 (size=12301) 2024-11-27T07:25:47,332 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/4c3784c82125428680e3a066b981b575 2024-11-27T07:25:47,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/eb4691d013de4737a1e99464a5623371 is 50, key is test_row_0/B:col10/1732692345924/Put/seqid=0 2024-11-27T07:25:47,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742147_1323 (size=12301) 2024-11-27T07:25:47,384 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/eb4691d013de4737a1e99464a5623371 2024-11-27T07:25:47,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/447b6a4a0125440fbfe2522a6534fde4 is 50, key is test_row_0/C:col10/1732692345924/Put/seqid=0 2024-11-27T07:25:47,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 81ee2c488a54a955269ce99517372c4b 2024-11-27T07:25:47,401 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. as already flushing 2024-11-27T07:25:47,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742148_1324 (size=12301) 2024-11-27T07:25:47,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:47,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59334 deadline: 1732692407454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:47,458 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:47,458 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:47,458 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:47,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:47,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59334 deadline: 1732692407569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:47,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:47,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:47,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:47,589 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/16457640c497411eb3a8fd6fe0bc9b08 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/16457640c497411eb3a8fd6fe0bc9b08 2024-11-27T07:25:47,593 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81ee2c488a54a955269ce99517372c4b/A of 81ee2c488a54a955269ce99517372c4b into 16457640c497411eb3a8fd6fe0bc9b08(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:25:47,593 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81ee2c488a54a955269ce99517372c4b: 2024-11-27T07:25:47,594 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., storeName=81ee2c488a54a955269ce99517372c4b/A, priority=13, startTime=1732692347149; duration=0sec 2024-11-27T07:25:47,594 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:25:47,594 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81ee2c488a54a955269ce99517372c4b:A 2024-11-27T07:25:47,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:47,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59334 deadline: 1732692407779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:47,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:47,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:47,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:47,844 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/447b6a4a0125440fbfe2522a6534fde4 2024-11-27T07:25:47,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/4c3784c82125428680e3a066b981b575 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/4c3784c82125428680e3a066b981b575 2024-11-27T07:25:47,851 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/4c3784c82125428680e3a066b981b575, entries=150, sequenceid=380, filesize=12.0 K 2024-11-27T07:25:47,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/eb4691d013de4737a1e99464a5623371 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/eb4691d013de4737a1e99464a5623371 2024-11-27T07:25:47,855 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/eb4691d013de4737a1e99464a5623371, entries=150, sequenceid=380, filesize=12.0 K 2024-11-27T07:25:47,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/447b6a4a0125440fbfe2522a6534fde4 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/447b6a4a0125440fbfe2522a6534fde4 2024-11-27T07:25:47,860 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/447b6a4a0125440fbfe2522a6534fde4, entries=150, sequenceid=380, filesize=12.0 K 2024-11-27T07:25:47,860 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 81ee2c488a54a955269ce99517372c4b in 573ms, sequenceid=380, compaction requested=false 2024-11-27T07:25:47,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 81ee2c488a54a955269ce99517372c4b: 2024-11-27T07:25:47,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:47,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-27T07:25:47,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-27T07:25:47,863 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-27T07:25:47,863 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8000 sec 2024-11-27T07:25:47,865 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.8040 sec 2024-11-27T07:25:47,970 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-27T07:25:48,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 81ee2c488a54a955269ce99517372c4b 2024-11-27T07:25:48,082 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T07:25:48,082 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 81ee2c488a54a955269ce99517372c4b 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-27T07:25:48,086 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/48c54b70ac5947b0a31c7a7911bc68d4 is 50, key is test_row_0/A:col10/1732692348081/Put/seqid=0 2024-11-27T07:25:48,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742149_1325 (size=14741) 2024-11-27T07:25:48,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:48,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59326 deadline: 1732692408111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:48,115 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:48,115 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:48,115 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:48,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:48,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59334 deadline: 1732692408115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:48,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:48,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:48,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:48,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:48,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1732692408115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:48,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:48,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:48,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:48,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:48,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59340 deadline: 1732692408169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:48,171 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 289954c9202d,42117,1732692259671 for 'test_row_2' in 81ee2c488a54a955269ce99517372c4b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T07:25:48,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:48,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:48,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:48,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:48,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59268 deadline: 1732692408179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:48,181 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 289954c9202d,42117,1732692259671 for 'test_row_1' in 81ee2c488a54a955269ce99517372c4b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T07:25:48,181 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:48,181 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:48,181 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:48,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-27T07:25:48,209 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T07:25:48,209 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T07:25:48,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-27T07:25:48,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-27T07:25:48,210 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T07:25:48,211 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T07:25:48,211 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T07:25:48,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:48,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59326 deadline: 1732692408219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:48,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:48,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:48,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:48,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:48,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:48,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59334 deadline: 1732692408229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:48,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1732692408229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:48,232 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:48,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:48,232 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:48,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:48,232 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:48,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:48,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-27T07:25:48,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-27T07:25:48,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:48,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. as already flushing 2024-11-27T07:25:48,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:48,363 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:48,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:48,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:48,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:48,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59326 deadline: 1732692408429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:48,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:48,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:48,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:48,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:48,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1732692408439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:48,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:48,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:48,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59334 deadline: 1732692408439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:48,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:48,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:48,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:48,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:48,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:48,490 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/48c54b70ac5947b0a31c7a7911bc68d4 2024-11-27T07:25:48,496 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/239921f2fffd4dcd991c05ae6c58820c is 50, key is test_row_0/B:col10/1732692348081/Put/seqid=0 2024-11-27T07:25:48,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742150_1326 (size=12301) 2024-11-27T07:25:48,516 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-27T07:25:48,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:48,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. as already flushing 2024-11-27T07:25:48,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:48,516 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:48,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:48,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:48,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-27T07:25:48,669 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-27T07:25:48,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:48,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. as already flushing 2024-11-27T07:25:48,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:48,669 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:48,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:48,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:48,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:48,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59326 deadline: 1732692408739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:48,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:48,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:48,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:48,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:48,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1732692408749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:48,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:48,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:48,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59334 deadline: 1732692408749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:48,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:48,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:48,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:48,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:48,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:48,822 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-27T07:25:48,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:48,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. as already flushing 2024-11-27T07:25:48,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:48,822 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:48,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:48,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:48,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-27T07:25:48,903 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/239921f2fffd4dcd991c05ae6c58820c 2024-11-27T07:25:48,908 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/3608fb89673e4ee6a1352604151056c2 is 50, key is test_row_0/C:col10/1732692348081/Put/seqid=0 2024-11-27T07:25:48,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742151_1327 (size=12301) 2024-11-27T07:25:48,974 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-27T07:25:48,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:48,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. as already flushing 2024-11-27T07:25:48,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:48,975 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:48,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:48,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:48,986 INFO [Thread-1132 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1132. 2024-11-27T07:25:48,987 DEBUG [Thread-1132 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T07:25:48,987 DEBUG [Thread-1132 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:25:48,987 DEBUG [Thread-1132 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:25:48,987 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T07:25:48,988 INFO [Thread-1138 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1138. 2024-11-27T07:25:48,988 DEBUG [Thread-1138 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T07:25:48,988 DEBUG [Thread-1138 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:25:48,988 DEBUG [Thread-1138 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:25:48,988 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T07:25:48,989 INFO [Thread-1134 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1134. 2024-11-27T07:25:48,989 DEBUG [Thread-1134 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T07:25:48,989 DEBUG [Thread-1134 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:25:48,989 DEBUG [Thread-1134 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:25:48,989 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T07:25:48,990 INFO [Thread-1140 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1140. 2024-11-27T07:25:48,991 DEBUG [Thread-1140 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T07:25:48,991 DEBUG [Thread-1140 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:25:48,991 DEBUG [Thread-1140 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:25:48,991 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T07:25:48,991 INFO [Thread-1136 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1136. 2024-11-27T07:25:48,991 DEBUG [Thread-1136 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T07:25:48,991 DEBUG [Thread-1136 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:25:48,991 DEBUG [Thread-1136 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:25:48,991 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T07:25:49,128 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-27T07:25:49,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:49,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. as already flushing 2024-11-27T07:25:49,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:49,129 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:49,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:49,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:49,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:49,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59326 deadline: 1732692409250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:49,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:49,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:49,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:49,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:49,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:49,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59308 deadline: 1732692409260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:49,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59334 deadline: 1732692409260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:49,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:49,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:49,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:49,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:49,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:49,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:25:49,285 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-27T07:25:49,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:49,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. as already flushing 2024-11-27T07:25:49,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:49,286 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:49,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:49,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:49,317 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/3608fb89673e4ee6a1352604151056c2 2024-11-27T07:25:49,325 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/48c54b70ac5947b0a31c7a7911bc68d4 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/48c54b70ac5947b0a31c7a7911bc68d4 2024-11-27T07:25:49,331 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/48c54b70ac5947b0a31c7a7911bc68d4, entries=200, sequenceid=396, filesize=14.4 K 2024-11-27T07:25:49,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/239921f2fffd4dcd991c05ae6c58820c as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/239921f2fffd4dcd991c05ae6c58820c 2024-11-27T07:25:49,337 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/239921f2fffd4dcd991c05ae6c58820c, entries=150, sequenceid=396, filesize=12.0 K 2024-11-27T07:25:49,338 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/3608fb89673e4ee6a1352604151056c2 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/3608fb89673e4ee6a1352604151056c2 2024-11-27T07:25:49,343 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/3608fb89673e4ee6a1352604151056c2, entries=150, sequenceid=396, filesize=12.0 K 2024-11-27T07:25:49,344 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 81ee2c488a54a955269ce99517372c4b in 1262ms, sequenceid=396, compaction requested=true 2024-11-27T07:25:49,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 81ee2c488a54a955269ce99517372c4b: 2024-11-27T07:25:49,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81ee2c488a54a955269ce99517372c4b:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T07:25:49,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:25:49,345 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:25:49,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81ee2c488a54a955269ce99517372c4b:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T07:25:49,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:25:49,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 81ee2c488a54a955269ce99517372c4b:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T07:25:49,345 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:25:49,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:25:49,346 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40229 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:25:49,346 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:25:49,346 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1541): 81ee2c488a54a955269ce99517372c4b/A is initiating minor compaction (all files) 2024-11-27T07:25:49,346 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 81ee2c488a54a955269ce99517372c4b/B is initiating minor compaction (all files) 2024-11-27T07:25:49,346 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81ee2c488a54a955269ce99517372c4b/A in TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:49,346 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81ee2c488a54a955269ce99517372c4b/B in TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:49,346 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/16457640c497411eb3a8fd6fe0bc9b08, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/4c3784c82125428680e3a066b981b575, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/48c54b70ac5947b0a31c7a7911bc68d4] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp, totalSize=39.3 K 2024-11-27T07:25:49,346 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/062aa03c3609481c8b61d8778f6af210, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/eb4691d013de4737a1e99464a5623371, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/239921f2fffd4dcd991c05ae6c58820c] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp, totalSize=36.9 K 2024-11-27T07:25:49,347 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16457640c497411eb3a8fd6fe0bc9b08, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1732692345552 2024-11-27T07:25:49,347 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 062aa03c3609481c8b61d8778f6af210, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1732692345552 2024-11-27T07:25:49,347 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting eb4691d013de4737a1e99464a5623371, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1732692345919 2024-11-27T07:25:49,347 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c3784c82125428680e3a066b981b575, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1732692345919 2024-11-27T07:25:49,348 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 239921f2fffd4dcd991c05ae6c58820c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1732692347447 2024-11-27T07:25:49,348 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48c54b70ac5947b0a31c7a7911bc68d4, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1732692347442 2024-11-27T07:25:49,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-27T07:25:49,358 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81ee2c488a54a955269ce99517372c4b#B#compaction#279 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:25:49,358 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81ee2c488a54a955269ce99517372c4b#A#compaction#280 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:25:49,358 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/f304fb8dbddd4817b19cfaccbedbf9e3 is 50, key is test_row_0/B:col10/1732692348081/Put/seqid=0 2024-11-27T07:25:49,358 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/e378930d71ca4a1188dd176381fa2c6a is 50, key is test_row_0/A:col10/1732692348081/Put/seqid=0 2024-11-27T07:25:49,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742153_1329 (size=13289) 2024-11-27T07:25:49,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742152_1328 (size=13289) 2024-11-27T07:25:49,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-27T07:25:49,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:49,443 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing 81ee2c488a54a955269ce99517372c4b 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-27T07:25:49,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/7ae4769535784f43a15783674c49199f is 50, key is test_row_0/A:col10/1732692348114/Put/seqid=0 2024-11-27T07:25:49,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742154_1330 (size=12301) 2024-11-27T07:25:49,772 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/f304fb8dbddd4817b19cfaccbedbf9e3 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/f304fb8dbddd4817b19cfaccbedbf9e3 2024-11-27T07:25:49,772 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/e378930d71ca4a1188dd176381fa2c6a as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/e378930d71ca4a1188dd176381fa2c6a 2024-11-27T07:25:49,778 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81ee2c488a54a955269ce99517372c4b/A of 81ee2c488a54a955269ce99517372c4b into e378930d71ca4a1188dd176381fa2c6a(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:25:49,778 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81ee2c488a54a955269ce99517372c4b/B of 81ee2c488a54a955269ce99517372c4b into f304fb8dbddd4817b19cfaccbedbf9e3(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:25:49,778 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81ee2c488a54a955269ce99517372c4b: 2024-11-27T07:25:49,778 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81ee2c488a54a955269ce99517372c4b: 2024-11-27T07:25:49,778 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., storeName=81ee2c488a54a955269ce99517372c4b/B, priority=13, startTime=1732692349345; duration=0sec 2024-11-27T07:25:49,778 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., storeName=81ee2c488a54a955269ce99517372c4b/A, priority=13, startTime=1732692349344; duration=0sec 2024-11-27T07:25:49,779 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:25:49,779 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81ee2c488a54a955269ce99517372c4b:B 2024-11-27T07:25:49,779 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:25:49,779 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81ee2c488a54a955269ce99517372c4b:A 2024-11-27T07:25:49,779 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:25:49,780 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:25:49,780 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 81ee2c488a54a955269ce99517372c4b/C is initiating minor compaction (all files) 2024-11-27T07:25:49,780 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 81ee2c488a54a955269ce99517372c4b/C in TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:49,780 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/3c9e545c249b4833bc61ff5c999c9b36, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/447b6a4a0125440fbfe2522a6534fde4, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/3608fb89673e4ee6a1352604151056c2] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp, totalSize=36.9 K 2024-11-27T07:25:49,780 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c9e545c249b4833bc61ff5c999c9b36, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1732692345552 2024-11-27T07:25:49,781 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 447b6a4a0125440fbfe2522a6534fde4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1732692345919 2024-11-27T07:25:49,781 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 3608fb89673e4ee6a1352604151056c2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1732692347447 2024-11-27T07:25:49,787 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 81ee2c488a54a955269ce99517372c4b#C#compaction#282 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:25:49,787 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/d8b6cedaac324f719475b64efa3f52c5 is 50, key is test_row_0/C:col10/1732692348081/Put/seqid=0 2024-11-27T07:25:49,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742155_1331 (size=13289) 2024-11-27T07:25:49,858 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/7ae4769535784f43a15783674c49199f 2024-11-27T07:25:49,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/c842033dce37482095fdc07e246f450e is 50, key is test_row_0/B:col10/1732692348114/Put/seqid=0 2024-11-27T07:25:49,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742156_1332 (size=12301) 2024-11-27T07:25:50,202 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/d8b6cedaac324f719475b64efa3f52c5 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/d8b6cedaac324f719475b64efa3f52c5 2024-11-27T07:25:50,210 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 81ee2c488a54a955269ce99517372c4b/C of 81ee2c488a54a955269ce99517372c4b into d8b6cedaac324f719475b64efa3f52c5(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:25:50,210 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 81ee2c488a54a955269ce99517372c4b: 2024-11-27T07:25:50,211 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., storeName=81ee2c488a54a955269ce99517372c4b/C, priority=13, startTime=1732692349345; duration=0sec 2024-11-27T07:25:50,211 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:25:50,211 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 81ee2c488a54a955269ce99517372c4b:C 2024-11-27T07:25:50,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 81ee2c488a54a955269ce99517372c4b 2024-11-27T07:25:50,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. as already flushing 2024-11-27T07:25:50,260 INFO [Thread-1123 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1123. 2024-11-27T07:25:50,260 DEBUG [Thread-1123 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T07:25:50,260 DEBUG [Thread-1123 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:25:50,261 DEBUG [Thread-1123 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:25:50,261 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T07:25:50,274 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/c842033dce37482095fdc07e246f450e 2024-11-27T07:25:50,282 INFO [Thread-1125 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1125. 2024-11-27T07:25:50,282 INFO [Thread-1127 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1127. 2024-11-27T07:25:50,282 DEBUG [Thread-1125 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T07:25:50,282 DEBUG [Thread-1125 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:25:50,282 DEBUG [Thread-1127 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T07:25:50,282 DEBUG [Thread-1127 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:25:50,282 DEBUG [Thread-1125 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:25:50,283 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T07:25:50,283 DEBUG [Thread-1127 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:25:50,283 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T07:25:50,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/cd970a2d755b42639ce7266654f90294 is 50, key is test_row_0/C:col10/1732692348114/Put/seqid=0 2024-11-27T07:25:50,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742157_1333 (size=12301) 2024-11-27T07:25:50,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-27T07:25:50,690 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/cd970a2d755b42639ce7266654f90294 2024-11-27T07:25:50,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/7ae4769535784f43a15783674c49199f as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/7ae4769535784f43a15783674c49199f 2024-11-27T07:25:50,704 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/7ae4769535784f43a15783674c49199f, entries=150, sequenceid=417, filesize=12.0 K 2024-11-27T07:25:50,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/c842033dce37482095fdc07e246f450e as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/c842033dce37482095fdc07e246f450e 2024-11-27T07:25:50,708 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/c842033dce37482095fdc07e246f450e, entries=150, sequenceid=417, filesize=12.0 K 2024-11-27T07:25:50,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/cd970a2d755b42639ce7266654f90294 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/cd970a2d755b42639ce7266654f90294 2024-11-27T07:25:50,712 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/cd970a2d755b42639ce7266654f90294, entries=150, sequenceid=417, filesize=12.0 K 2024-11-27T07:25:50,713 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=20.13 KB/20610 for 81ee2c488a54a955269ce99517372c4b in 1271ms, sequenceid=417, compaction requested=false 2024-11-27T07:25:50,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for 81ee2c488a54a955269ce99517372c4b: 2024-11-27T07:25:50,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:50,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-27T07:25:50,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-27T07:25:50,716 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-27T07:25:50,716 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5030 sec 2024-11-27T07:25:50,717 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 2.5070 sec 2024-11-27T07:25:52,214 INFO [Thread-1129 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1129. 2024-11-27T07:25:52,214 INFO [Thread-1121 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1121. 2024-11-27T07:25:52,214 DEBUG [Thread-1129 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T07:25:52,214 DEBUG [Thread-1121 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T07:25:52,215 DEBUG [Thread-1129 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:25:52,215 DEBUG [Thread-1121 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:25:52,215 DEBUG [Thread-1129 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:25:52,215 DEBUG [Thread-1121 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:25:52,215 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T07:25:52,215 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T07:25:52,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-27T07:25:52,379 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T07:25:52,379 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-27T07:25:52,379 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 67 2024-11-27T07:25:52,379 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-11-27T07:25:52,379 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 73 2024-11-27T07:25:52,379 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-11-27T07:25:52,379 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-11-27T07:25:52,379 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-27T07:25:52,379 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3792 2024-11-27T07:25:52,379 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3719 2024-11-27T07:25:52,379 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3790 2024-11-27T07:25:52,379 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3841 2024-11-27T07:25:52,379 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3727 2024-11-27T07:25:52,379 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-27T07:25:52,379 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T07:25:52,380 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T07:25:52,380 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:25:52,380 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:25:52,380 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T07:25:52,380 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-27T07:25:52,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-27T07:25:52,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-27T07:25:52,383 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732692352383"}]},"ts":"1732692352383"} 2024-11-27T07:25:52,385 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-27T07:25:52,385 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-27T07:25:52,385 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-27T07:25:52,387 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=81ee2c488a54a955269ce99517372c4b, UNASSIGN}] 2024-11-27T07:25:52,388 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=81ee2c488a54a955269ce99517372c4b, UNASSIGN 2024-11-27T07:25:52,388 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=81ee2c488a54a955269ce99517372c4b, regionState=CLOSING, regionLocation=289954c9202d,42117,1732692259671 2024-11-27T07:25:52,390 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=81ee2c488a54a955269ce99517372c4b, UNASSIGN because future has completed 2024-11-27T07:25:52,390 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T07:25:52,390 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure 81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671}] 2024-11-27T07:25:52,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-27T07:25:52,544 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(122): Close 81ee2c488a54a955269ce99517372c4b 2024-11-27T07:25:52,544 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T07:25:52,544 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1722): Closing 81ee2c488a54a955269ce99517372c4b, disabling compactions & flushes 2024-11-27T07:25:52,544 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:52,545 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:52,545 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. after waiting 0 ms 2024-11-27T07:25:52,545 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:52,545 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(2902): Flushing 81ee2c488a54a955269ce99517372c4b 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-27T07:25:52,554 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/d81c972da16e4731849b0c2a96b30f1f is 50, key is test_row_0/A:col10/1732692350280/Put/seqid=0 2024-11-27T07:25:52,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742158_1334 (size=12301) 2024-11-27T07:25:52,559 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/d81c972da16e4731849b0c2a96b30f1f 2024-11-27T07:25:52,566 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/bee0fdd117e84aa3824345d90d49359d is 50, key is test_row_0/B:col10/1732692350280/Put/seqid=0 2024-11-27T07:25:52,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742159_1335 (size=12301) 2024-11-27T07:25:52,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-27T07:25:52,971 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/bee0fdd117e84aa3824345d90d49359d 2024-11-27T07:25:52,986 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/e4aa7ae222394df2b65f65bdf2e469db is 50, key is test_row_0/C:col10/1732692350280/Put/seqid=0 2024-11-27T07:25:52,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742160_1336 (size=12301) 2024-11-27T07:25:53,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-27T07:25:53,392 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/e4aa7ae222394df2b65f65bdf2e469db 2024-11-27T07:25:53,395 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/A/d81c972da16e4731849b0c2a96b30f1f as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/d81c972da16e4731849b0c2a96b30f1f 2024-11-27T07:25:53,398 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/d81c972da16e4731849b0c2a96b30f1f, entries=150, sequenceid=428, filesize=12.0 K 2024-11-27T07:25:53,399 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/B/bee0fdd117e84aa3824345d90d49359d as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/bee0fdd117e84aa3824345d90d49359d 2024-11-27T07:25:53,402 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/bee0fdd117e84aa3824345d90d49359d, entries=150, sequenceid=428, filesize=12.0 K 2024-11-27T07:25:53,403 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/.tmp/C/e4aa7ae222394df2b65f65bdf2e469db as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/e4aa7ae222394df2b65f65bdf2e469db 2024-11-27T07:25:53,406 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/e4aa7ae222394df2b65f65bdf2e469db, entries=150, sequenceid=428, filesize=12.0 K 2024-11-27T07:25:53,407 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 81ee2c488a54a955269ce99517372c4b in 861ms, sequenceid=428, compaction requested=true 2024-11-27T07:25:53,407 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/144330b3b9554335b20af695f36aa810, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/14f86348aa704449b4d7cac0a78c97c1, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/89e21cbcf5014318aa5bd91f194947b7, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/3adf4dd86d674f4fa37cb467e6ea44ca, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/f2ce488fdf0f44ad897de95134ed3aa7, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/a735557a9fb348da889c980b985d97af, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/2ff90d4d98d94d59a9141423f91b2e21, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/76970dca5ed34cb7b0f528be88fb378c, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/7f4791b0bba245dba3eeb559edca9c94, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/2013c82fabcd4ecb924b80aaae329026, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/24f26cc9e74f41e3ad23e44149be838f, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/0396eacaa36647528c26c535aa147621, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/0bc1ecd03d6048c2a88b69377e677a38, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/478011dcba0549749a47d070e4b401c0, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/3b5004f7da0e42faaad675fde7d6f454, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/1644044bf4d640e5a877d4ef07caecac, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/48cbceaec0614b31a79b83854e843699, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/17f24cf244b24f1abf80d1eca6d74c79, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/5e32336eff4a4c7fac3d0965de3dfbb0, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/2e6f0941557c4a6f8018b35acdc9d95f, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/34c965517d784550a67310c096349b77, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/0edfc3f986264752adc767bc079a972f, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/750d672226644223959bfa26bfc7bb26, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/f7489dd2edcd4ff99f7492ffe7098fa3, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/501d81e8a5ba4e2fb4c7d9f687be5a20, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/3edf4287eb8b48e4a075d3adb4a54eba, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/16457640c497411eb3a8fd6fe0bc9b08, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/4c3784c82125428680e3a066b981b575, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/48c54b70ac5947b0a31c7a7911bc68d4] to archive 2024-11-27T07:25:53,408 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T07:25:53,410 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/144330b3b9554335b20af695f36aa810 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/144330b3b9554335b20af695f36aa810 2024-11-27T07:25:53,411 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/14f86348aa704449b4d7cac0a78c97c1 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/14f86348aa704449b4d7cac0a78c97c1 2024-11-27T07:25:53,413 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/89e21cbcf5014318aa5bd91f194947b7 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/89e21cbcf5014318aa5bd91f194947b7 2024-11-27T07:25:53,414 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/3adf4dd86d674f4fa37cb467e6ea44ca to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/3adf4dd86d674f4fa37cb467e6ea44ca 2024-11-27T07:25:53,415 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/f2ce488fdf0f44ad897de95134ed3aa7 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/f2ce488fdf0f44ad897de95134ed3aa7 2024-11-27T07:25:53,417 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/a735557a9fb348da889c980b985d97af to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/a735557a9fb348da889c980b985d97af 2024-11-27T07:25:53,418 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/2ff90d4d98d94d59a9141423f91b2e21 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/2ff90d4d98d94d59a9141423f91b2e21 2024-11-27T07:25:53,420 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/76970dca5ed34cb7b0f528be88fb378c to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/76970dca5ed34cb7b0f528be88fb378c 2024-11-27T07:25:53,422 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/7f4791b0bba245dba3eeb559edca9c94 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/7f4791b0bba245dba3eeb559edca9c94 2024-11-27T07:25:53,424 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/2013c82fabcd4ecb924b80aaae329026 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/2013c82fabcd4ecb924b80aaae329026 2024-11-27T07:25:53,425 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/24f26cc9e74f41e3ad23e44149be838f to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/24f26cc9e74f41e3ad23e44149be838f 2024-11-27T07:25:53,426 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/0396eacaa36647528c26c535aa147621 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/0396eacaa36647528c26c535aa147621 2024-11-27T07:25:53,428 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/0bc1ecd03d6048c2a88b69377e677a38 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/0bc1ecd03d6048c2a88b69377e677a38 2024-11-27T07:25:53,429 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/478011dcba0549749a47d070e4b401c0 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/478011dcba0549749a47d070e4b401c0 2024-11-27T07:25:53,431 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/3b5004f7da0e42faaad675fde7d6f454 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/3b5004f7da0e42faaad675fde7d6f454 2024-11-27T07:25:53,432 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/1644044bf4d640e5a877d4ef07caecac to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/1644044bf4d640e5a877d4ef07caecac 2024-11-27T07:25:53,433 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/48cbceaec0614b31a79b83854e843699 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/48cbceaec0614b31a79b83854e843699 2024-11-27T07:25:53,434 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/17f24cf244b24f1abf80d1eca6d74c79 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/17f24cf244b24f1abf80d1eca6d74c79 2024-11-27T07:25:53,435 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/5e32336eff4a4c7fac3d0965de3dfbb0 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/5e32336eff4a4c7fac3d0965de3dfbb0 2024-11-27T07:25:53,436 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/2e6f0941557c4a6f8018b35acdc9d95f to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/2e6f0941557c4a6f8018b35acdc9d95f 2024-11-27T07:25:53,437 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/34c965517d784550a67310c096349b77 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/34c965517d784550a67310c096349b77 2024-11-27T07:25:53,438 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/0edfc3f986264752adc767bc079a972f to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/0edfc3f986264752adc767bc079a972f 2024-11-27T07:25:53,439 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/750d672226644223959bfa26bfc7bb26 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/750d672226644223959bfa26bfc7bb26 2024-11-27T07:25:53,440 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/f7489dd2edcd4ff99f7492ffe7098fa3 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/f7489dd2edcd4ff99f7492ffe7098fa3 2024-11-27T07:25:53,441 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/501d81e8a5ba4e2fb4c7d9f687be5a20 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/501d81e8a5ba4e2fb4c7d9f687be5a20 2024-11-27T07:25:53,442 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/3edf4287eb8b48e4a075d3adb4a54eba to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/3edf4287eb8b48e4a075d3adb4a54eba 2024-11-27T07:25:53,442 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/16457640c497411eb3a8fd6fe0bc9b08 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/16457640c497411eb3a8fd6fe0bc9b08 2024-11-27T07:25:53,443 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/4c3784c82125428680e3a066b981b575 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/4c3784c82125428680e3a066b981b575 2024-11-27T07:25:53,444 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/48c54b70ac5947b0a31c7a7911bc68d4 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/48c54b70ac5947b0a31c7a7911bc68d4 2024-11-27T07:25:53,445 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/38564da6bc1044918d34852e6e66e773, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/1ca2d866ad474e18a09010c44232404f, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/3eb82172993e4559ac4b18919d940945, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/4c0b025b161d4a1386b1d6f16ca9a039, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/f89f409dff994323990043c02ca3d15a, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/c71ae23fecaa4b8b9097ac0ec73e9e96, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/a6550861808040889ade3e6059e5e781, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/302e213fd0cd4f1d8157d388bd82cf93, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/cc3dca164e004fceb603c972afbca92d, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/89e0f68310814734967444b2eb3a912d, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/4fd6849935d343dc99efa8608dd74f46, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/f7b527acf1ba46dca80aa4a9ad4b3c98, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/b29952c5b06942bd9316b1568c4dd0ef, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/1ac2e23b2c2949c19cb102671475ff35, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/50dec500678e4be48ce54a7d37f7ba69, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/5344398c28e64dae813f295a40c1dfd4, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/a2f39c2d0fe648698f5b86c30ed47034, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/c7c19c39f64b4a578acdb97504b62ba2, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/8c0be216c09a475cba4763fb7326544f, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/77e53d92d7de4ee9ade3876e0a19ee06, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/acabf716411a48d98f97da92c16df777, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/c37ccb40a6944a40982d3c2f5dc29565, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/f7297aff273b43488169d7d878358f84, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/81fe68fe690d4ac191121a68c154ae8e, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/69ee67ff9cc1410291ee1dc4cd560e26, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/062aa03c3609481c8b61d8778f6af210, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/6fed829faaa2424d92a4baf14e7e661c, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/eb4691d013de4737a1e99464a5623371, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/239921f2fffd4dcd991c05ae6c58820c] to archive 2024-11-27T07:25:53,446 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T07:25:53,448 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/38564da6bc1044918d34852e6e66e773 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/38564da6bc1044918d34852e6e66e773 2024-11-27T07:25:53,448 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/1ca2d866ad474e18a09010c44232404f to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/1ca2d866ad474e18a09010c44232404f 2024-11-27T07:25:53,449 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/3eb82172993e4559ac4b18919d940945 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/3eb82172993e4559ac4b18919d940945 2024-11-27T07:25:53,450 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/4c0b025b161d4a1386b1d6f16ca9a039 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/4c0b025b161d4a1386b1d6f16ca9a039 2024-11-27T07:25:53,451 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/f89f409dff994323990043c02ca3d15a to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/f89f409dff994323990043c02ca3d15a 2024-11-27T07:25:53,452 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/c71ae23fecaa4b8b9097ac0ec73e9e96 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/c71ae23fecaa4b8b9097ac0ec73e9e96 2024-11-27T07:25:53,453 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/a6550861808040889ade3e6059e5e781 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/a6550861808040889ade3e6059e5e781 2024-11-27T07:25:53,454 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/302e213fd0cd4f1d8157d388bd82cf93 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/302e213fd0cd4f1d8157d388bd82cf93 2024-11-27T07:25:53,455 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/cc3dca164e004fceb603c972afbca92d to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/cc3dca164e004fceb603c972afbca92d 2024-11-27T07:25:53,455 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/89e0f68310814734967444b2eb3a912d to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/89e0f68310814734967444b2eb3a912d 2024-11-27T07:25:53,457 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/4fd6849935d343dc99efa8608dd74f46 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/4fd6849935d343dc99efa8608dd74f46 2024-11-27T07:25:53,457 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/f7b527acf1ba46dca80aa4a9ad4b3c98 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/f7b527acf1ba46dca80aa4a9ad4b3c98 2024-11-27T07:25:53,459 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/b29952c5b06942bd9316b1568c4dd0ef to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/b29952c5b06942bd9316b1568c4dd0ef 2024-11-27T07:25:53,460 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/1ac2e23b2c2949c19cb102671475ff35 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/1ac2e23b2c2949c19cb102671475ff35 2024-11-27T07:25:53,461 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/50dec500678e4be48ce54a7d37f7ba69 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/50dec500678e4be48ce54a7d37f7ba69 2024-11-27T07:25:53,461 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/5344398c28e64dae813f295a40c1dfd4 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/5344398c28e64dae813f295a40c1dfd4 2024-11-27T07:25:53,462 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/a2f39c2d0fe648698f5b86c30ed47034 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/a2f39c2d0fe648698f5b86c30ed47034 2024-11-27T07:25:53,463 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/c7c19c39f64b4a578acdb97504b62ba2 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/c7c19c39f64b4a578acdb97504b62ba2 2024-11-27T07:25:53,464 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/8c0be216c09a475cba4763fb7326544f to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/8c0be216c09a475cba4763fb7326544f 2024-11-27T07:25:53,464 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/77e53d92d7de4ee9ade3876e0a19ee06 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/77e53d92d7de4ee9ade3876e0a19ee06 2024-11-27T07:25:53,465 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/acabf716411a48d98f97da92c16df777 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/acabf716411a48d98f97da92c16df777 2024-11-27T07:25:53,466 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/c37ccb40a6944a40982d3c2f5dc29565 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/c37ccb40a6944a40982d3c2f5dc29565 2024-11-27T07:25:53,467 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/f7297aff273b43488169d7d878358f84 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/f7297aff273b43488169d7d878358f84 2024-11-27T07:25:53,467 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/81fe68fe690d4ac191121a68c154ae8e to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/81fe68fe690d4ac191121a68c154ae8e 2024-11-27T07:25:53,468 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/69ee67ff9cc1410291ee1dc4cd560e26 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/69ee67ff9cc1410291ee1dc4cd560e26 2024-11-27T07:25:53,469 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/062aa03c3609481c8b61d8778f6af210 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/062aa03c3609481c8b61d8778f6af210 2024-11-27T07:25:53,470 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/6fed829faaa2424d92a4baf14e7e661c to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/6fed829faaa2424d92a4baf14e7e661c 2024-11-27T07:25:53,470 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/eb4691d013de4737a1e99464a5623371 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/eb4691d013de4737a1e99464a5623371 2024-11-27T07:25:53,471 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/239921f2fffd4dcd991c05ae6c58820c to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/239921f2fffd4dcd991c05ae6c58820c 2024-11-27T07:25:53,472 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/b7f2e755f27a45be8b2fbd609bcbdb27, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/78698476588646b38f69c7bcca09e1c2, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/6416c66c71804937a58cca053720bfc2, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/cdec4d8b0b70426fa32dc011a9f64ef4, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/002d440bf6a54d44854c3e4a3d5d6c37, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/c741face4aba407ca4f45e19870fcbbb, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/39ea6c02dc1f4c0cbbc35c5b35c7fc41, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/d8d757558f1f4b6f8eb6046b1ac3be87, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/8742f5fd16df455c9d26d37aef80fb9c, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/ae9744423683402ab0282a7169800cd0, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/e54bd2f77db246a1816bf09c0fa74712, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/d897fca4019a4e4290678762bee4b8be, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/e409e0fa9099429cbf87fc385b1823fe, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/5a9c539e17464a048705ac7a0c4b90b3, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/d937a88257784bff948ac15a833fb0c4, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/0979171387b34c95922f844d6aa5faac, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/c09dd39ddbb84157a180f42e2d0b5d76, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/b3bcbef2226f4e7e86c17756f3640cd5, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/81b2149867654ae7b95e50dceb0733db, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/89be84149fce4092855bd56f0f45c0a8, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/1e8103d1b02d480d820019c3d28d5c1b, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/11da7fd827f1402891820c1bf867dc62, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/0975e37d6c7c4074957506ebb6610ff6, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/0bbb11043f7d4abc8c71064d2f3fefc2, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/186e81a15a994457a9b3bbb867a4d9ec, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/3c9e545c249b4833bc61ff5c999c9b36, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/9c2a1646ad094cd38f61292545bc99d2, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/447b6a4a0125440fbfe2522a6534fde4, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/3608fb89673e4ee6a1352604151056c2] to archive 2024-11-27T07:25:53,473 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T07:25:53,474 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/b7f2e755f27a45be8b2fbd609bcbdb27 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/b7f2e755f27a45be8b2fbd609bcbdb27 2024-11-27T07:25:53,475 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/78698476588646b38f69c7bcca09e1c2 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/78698476588646b38f69c7bcca09e1c2 2024-11-27T07:25:53,476 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/6416c66c71804937a58cca053720bfc2 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/6416c66c71804937a58cca053720bfc2 2024-11-27T07:25:53,476 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/cdec4d8b0b70426fa32dc011a9f64ef4 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/cdec4d8b0b70426fa32dc011a9f64ef4 2024-11-27T07:25:53,477 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/002d440bf6a54d44854c3e4a3d5d6c37 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/002d440bf6a54d44854c3e4a3d5d6c37 2024-11-27T07:25:53,478 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/c741face4aba407ca4f45e19870fcbbb to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/c741face4aba407ca4f45e19870fcbbb 2024-11-27T07:25:53,479 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/39ea6c02dc1f4c0cbbc35c5b35c7fc41 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/39ea6c02dc1f4c0cbbc35c5b35c7fc41 2024-11-27T07:25:53,479 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/d8d757558f1f4b6f8eb6046b1ac3be87 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/d8d757558f1f4b6f8eb6046b1ac3be87 2024-11-27T07:25:53,480 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/8742f5fd16df455c9d26d37aef80fb9c to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/8742f5fd16df455c9d26d37aef80fb9c 2024-11-27T07:25:53,481 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/ae9744423683402ab0282a7169800cd0 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/ae9744423683402ab0282a7169800cd0 2024-11-27T07:25:53,481 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/e54bd2f77db246a1816bf09c0fa74712 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/e54bd2f77db246a1816bf09c0fa74712 2024-11-27T07:25:53,482 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/d897fca4019a4e4290678762bee4b8be to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/d897fca4019a4e4290678762bee4b8be 2024-11-27T07:25:53,483 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/e409e0fa9099429cbf87fc385b1823fe to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/e409e0fa9099429cbf87fc385b1823fe 2024-11-27T07:25:53,484 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/5a9c539e17464a048705ac7a0c4b90b3 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/5a9c539e17464a048705ac7a0c4b90b3 2024-11-27T07:25:53,485 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/d937a88257784bff948ac15a833fb0c4 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/d937a88257784bff948ac15a833fb0c4 2024-11-27T07:25:53,485 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/0979171387b34c95922f844d6aa5faac to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/0979171387b34c95922f844d6aa5faac 2024-11-27T07:25:53,486 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/c09dd39ddbb84157a180f42e2d0b5d76 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/c09dd39ddbb84157a180f42e2d0b5d76 2024-11-27T07:25:53,487 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/b3bcbef2226f4e7e86c17756f3640cd5 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/b3bcbef2226f4e7e86c17756f3640cd5 2024-11-27T07:25:53,488 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/81b2149867654ae7b95e50dceb0733db to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/81b2149867654ae7b95e50dceb0733db 2024-11-27T07:25:53,488 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/89be84149fce4092855bd56f0f45c0a8 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/89be84149fce4092855bd56f0f45c0a8 2024-11-27T07:25:53,489 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/1e8103d1b02d480d820019c3d28d5c1b to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/1e8103d1b02d480d820019c3d28d5c1b 2024-11-27T07:25:53,490 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/11da7fd827f1402891820c1bf867dc62 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/11da7fd827f1402891820c1bf867dc62 2024-11-27T07:25:53,491 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/0975e37d6c7c4074957506ebb6610ff6 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/0975e37d6c7c4074957506ebb6610ff6 2024-11-27T07:25:53,492 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/0bbb11043f7d4abc8c71064d2f3fefc2 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/0bbb11043f7d4abc8c71064d2f3fefc2 2024-11-27T07:25:53,493 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/186e81a15a994457a9b3bbb867a4d9ec to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/186e81a15a994457a9b3bbb867a4d9ec 2024-11-27T07:25:53,493 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/3c9e545c249b4833bc61ff5c999c9b36 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/3c9e545c249b4833bc61ff5c999c9b36 2024-11-27T07:25:53,494 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/9c2a1646ad094cd38f61292545bc99d2 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/9c2a1646ad094cd38f61292545bc99d2 2024-11-27T07:25:53,495 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/447b6a4a0125440fbfe2522a6534fde4 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/447b6a4a0125440fbfe2522a6534fde4 2024-11-27T07:25:53,496 DEBUG [StoreCloser-TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/3608fb89673e4ee6a1352604151056c2 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/3608fb89673e4ee6a1352604151056c2 2024-11-27T07:25:53,499 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/recovered.edits/431.seqid, newMaxSeqId=431, maxSeqId=1 2024-11-27T07:25:53,500 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. 2024-11-27T07:25:53,500 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1676): Region close journal for 81ee2c488a54a955269ce99517372c4b: Waiting for close lock at 1732692352544Running coprocessor pre-close hooks at 1732692352544Disabling compacts and flushes for region at 1732692352544Disabling writes for close at 1732692352545 (+1 ms)Obtaining lock to block concurrent updates at 1732692352545Preparing flush snapshotting stores in 81ee2c488a54a955269ce99517372c4b at 1732692352545Finished memstore snapshotting TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1732692352546 (+1 ms)Flushing stores of TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b. at 1732692352547 (+1 ms)Flushing 81ee2c488a54a955269ce99517372c4b/A: creating writer at 1732692352547Flushing 81ee2c488a54a955269ce99517372c4b/A: appending metadata at 1732692352553 (+6 ms)Flushing 81ee2c488a54a955269ce99517372c4b/A: closing flushed file at 1732692352553Flushing 81ee2c488a54a955269ce99517372c4b/B: creating writer at 1732692352563 (+10 ms)Flushing 81ee2c488a54a955269ce99517372c4b/B: appending metadata at 1732692352565 (+2 ms)Flushing 81ee2c488a54a955269ce99517372c4b/B: closing flushed file at 1732692352565Flushing 81ee2c488a54a955269ce99517372c4b/C: creating writer at 1732692352980 (+415 ms)Flushing 81ee2c488a54a955269ce99517372c4b/C: appending metadata at 1732692352985 (+5 ms)Flushing 81ee2c488a54a955269ce99517372c4b/C: closing flushed file at 1732692352985Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@304e20a2: reopening flushed file at 1732692353395 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2b513637: reopening flushed file at 1732692353398 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2b3a6b23: reopening flushed file at 1732692353402 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 81ee2c488a54a955269ce99517372c4b in 861ms, sequenceid=428, compaction requested=true at 1732692353407 (+5 ms)Writing region close event to WAL at 1732692353497 (+90 ms)Running coprocessor post-close hooks at 1732692353500 (+3 ms)Closed at 1732692353500 2024-11-27T07:25:53,502 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(157): Closed 81ee2c488a54a955269ce99517372c4b 2024-11-27T07:25:53,502 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=81ee2c488a54a955269ce99517372c4b, regionState=CLOSED 2024-11-27T07:25:53,503 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure 81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 because future has completed 2024-11-27T07:25:53,505 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-27T07:25:53,505 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; CloseRegionProcedure 81ee2c488a54a955269ce99517372c4b, server=289954c9202d,42117,1732692259671 in 1.1140 sec 2024-11-27T07:25:53,506 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-11-27T07:25:53,506 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=81ee2c488a54a955269ce99517372c4b, UNASSIGN in 1.1180 sec 2024-11-27T07:25:53,508 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-27T07:25:53,508 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.1220 sec 2024-11-27T07:25:53,509 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732692353509"}]},"ts":"1732692353509"} 2024-11-27T07:25:53,510 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-27T07:25:53,510 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-27T07:25:53,511 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.1310 sec 2024-11-27T07:25:53,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-27T07:25:53,519 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-27T07:25:53,520 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-27T07:25:53,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T07:25:53,522 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=88, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T07:25:53,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-27T07:25:53,522 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=88, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T07:25:53,524 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b 2024-11-27T07:25:53,526 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A, FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B, FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C, FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/recovered.edits] 2024-11-27T07:25:53,529 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/7ae4769535784f43a15783674c49199f to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/7ae4769535784f43a15783674c49199f 2024-11-27T07:25:53,530 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/d81c972da16e4731849b0c2a96b30f1f to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/d81c972da16e4731849b0c2a96b30f1f 2024-11-27T07:25:53,531 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/e378930d71ca4a1188dd176381fa2c6a to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/A/e378930d71ca4a1188dd176381fa2c6a 2024-11-27T07:25:53,532 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/bee0fdd117e84aa3824345d90d49359d to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/bee0fdd117e84aa3824345d90d49359d 2024-11-27T07:25:53,533 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/c842033dce37482095fdc07e246f450e to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/c842033dce37482095fdc07e246f450e 2024-11-27T07:25:53,534 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/f304fb8dbddd4817b19cfaccbedbf9e3 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/B/f304fb8dbddd4817b19cfaccbedbf9e3 2024-11-27T07:25:53,536 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/cd970a2d755b42639ce7266654f90294 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/cd970a2d755b42639ce7266654f90294 2024-11-27T07:25:53,537 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/d8b6cedaac324f719475b64efa3f52c5 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/d8b6cedaac324f719475b64efa3f52c5 2024-11-27T07:25:53,538 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/e4aa7ae222394df2b65f65bdf2e469db to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/C/e4aa7ae222394df2b65f65bdf2e469db 2024-11-27T07:25:53,541 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/recovered.edits/431.seqid to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b/recovered.edits/431.seqid 2024-11-27T07:25:53,541 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/81ee2c488a54a955269ce99517372c4b 2024-11-27T07:25:53,541 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-27T07:25:53,543 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=88, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T07:25:53,546 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-27T07:25:53,548 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-27T07:25:53,550 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=88, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T07:25:53,550 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-27T07:25:53,550 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732692353550"}]},"ts":"9223372036854775807"} 2024-11-27T07:25:53,552 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-27T07:25:53,552 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 81ee2c488a54a955269ce99517372c4b, NAME => 'TestAcidGuarantees,,1732692326691.81ee2c488a54a955269ce99517372c4b.', STARTKEY => '', ENDKEY => ''}] 2024-11-27T07:25:53,552 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-27T07:25:53,552 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732692353552"}]},"ts":"9223372036854775807"} 2024-11-27T07:25:53,554 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-27T07:25:53,555 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=88, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T07:25:53,556 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 35 msec 2024-11-27T07:25:53,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-27T07:25:53,629 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-27T07:25:53,631 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-27T07:25:53,638 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=240 (was 238) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-730273825_22 at /127.0.0.1:47174 [Waiting for operation #491] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1984458721_22 at /127.0.0.1:39988 [Waiting for operation #100] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=481 (was 477) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4457 (was 4386) - AvailableMemoryMB LEAK? - 2024-11-27T07:25:53,647 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=240, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=481, ProcessCount=11, AvailableMemoryMB=4457 2024-11-27T07:25:53,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-27T07:25:53,649 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T07:25:53,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-27T07:25:53,651 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T07:25:53,651 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:25:53,652 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 89 2024-11-27T07:25:53,652 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T07:25:53,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-27T07:25:53,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742161_1337 (size=959) 2024-11-27T07:25:53,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-27T07:25:53,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-27T07:25:54,065 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4 2024-11-27T07:25:54,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742162_1338 (size=53) 2024-11-27T07:25:54,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-27T07:25:54,475 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T07:25:54,475 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 9cd6e5b712012f5d5b13dc2bcef88d1f, disabling compactions & flushes 2024-11-27T07:25:54,476 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:25:54,476 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:25:54,476 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. after waiting 0 ms 2024-11-27T07:25:54,476 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:25:54,476 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:25:54,476 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9cd6e5b712012f5d5b13dc2bcef88d1f: Waiting for close lock at 1732692354475Disabling compacts and flushes for region at 1732692354475Disabling writes for close at 1732692354476 (+1 ms)Writing region close event to WAL at 1732692354476Closed at 1732692354476 2024-11-27T07:25:54,478 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T07:25:54,479 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732692354478"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732692354478"}]},"ts":"1732692354478"} 2024-11-27T07:25:54,482 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T07:25:54,483 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T07:25:54,484 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732692354483"}]},"ts":"1732692354483"} 2024-11-27T07:25:54,485 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-27T07:25:54,486 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9cd6e5b712012f5d5b13dc2bcef88d1f, ASSIGN}] 2024-11-27T07:25:54,487 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9cd6e5b712012f5d5b13dc2bcef88d1f, ASSIGN 2024-11-27T07:25:54,488 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9cd6e5b712012f5d5b13dc2bcef88d1f, ASSIGN; state=OFFLINE, location=289954c9202d,42117,1732692259671; forceNewPlan=false, retain=false 2024-11-27T07:25:54,639 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=9cd6e5b712012f5d5b13dc2bcef88d1f, regionState=OPENING, regionLocation=289954c9202d,42117,1732692259671 2024-11-27T07:25:54,641 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9cd6e5b712012f5d5b13dc2bcef88d1f, ASSIGN because future has completed 2024-11-27T07:25:54,641 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671}] 2024-11-27T07:25:54,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-27T07:25:54,800 INFO [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:25:54,800 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7752): Opening region: {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} 2024-11-27T07:25:54,801 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:25:54,801 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T07:25:54,801 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7794): checking encryption for 9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:25:54,801 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7797): checking classloading for 9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:25:54,804 INFO [StoreOpener-9cd6e5b712012f5d5b13dc2bcef88d1f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:25:54,807 INFO [StoreOpener-9cd6e5b712012f5d5b13dc2bcef88d1f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9cd6e5b712012f5d5b13dc2bcef88d1f columnFamilyName A 2024-11-27T07:25:54,807 DEBUG [StoreOpener-9cd6e5b712012f5d5b13dc2bcef88d1f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:25:54,808 INFO [StoreOpener-9cd6e5b712012f5d5b13dc2bcef88d1f-1 {}] regionserver.HStore(327): Store=9cd6e5b712012f5d5b13dc2bcef88d1f/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T07:25:54,808 INFO [StoreOpener-9cd6e5b712012f5d5b13dc2bcef88d1f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:25:54,810 INFO [StoreOpener-9cd6e5b712012f5d5b13dc2bcef88d1f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9cd6e5b712012f5d5b13dc2bcef88d1f columnFamilyName B 2024-11-27T07:25:54,810 DEBUG [StoreOpener-9cd6e5b712012f5d5b13dc2bcef88d1f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:25:54,810 INFO [StoreOpener-9cd6e5b712012f5d5b13dc2bcef88d1f-1 {}] regionserver.HStore(327): Store=9cd6e5b712012f5d5b13dc2bcef88d1f/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T07:25:54,811 INFO [StoreOpener-9cd6e5b712012f5d5b13dc2bcef88d1f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:25:54,812 INFO [StoreOpener-9cd6e5b712012f5d5b13dc2bcef88d1f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9cd6e5b712012f5d5b13dc2bcef88d1f columnFamilyName C 2024-11-27T07:25:54,812 DEBUG [StoreOpener-9cd6e5b712012f5d5b13dc2bcef88d1f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:25:54,813 INFO [StoreOpener-9cd6e5b712012f5d5b13dc2bcef88d1f-1 {}] regionserver.HStore(327): Store=9cd6e5b712012f5d5b13dc2bcef88d1f/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T07:25:54,813 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1038): replaying wal for 9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:25:54,814 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:25:54,815 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:25:54,817 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1048): stopping wal replay for 9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:25:54,817 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1060): Cleaning up temporary data for 9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:25:54,817 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T07:25:54,819 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1093): writing seq id for 9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:25:54,821 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T07:25:54,821 INFO [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1114): Opened 9cd6e5b712012f5d5b13dc2bcef88d1f; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60006442, jitterRate=-0.10583433508872986}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T07:25:54,821 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:25:54,822 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1006): Region open journal for 9cd6e5b712012f5d5b13dc2bcef88d1f: Running coprocessor pre-open hook at 1732692354801Writing region info on filesystem at 1732692354801Initializing all the Stores at 1732692354803 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732692354803Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732692354804 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732692354804Cleaning up temporary data from old regions at 1732692354817 (+13 ms)Running coprocessor post-open hooks at 1732692354821 (+4 ms)Region opened successfully at 1732692354822 (+1 ms) 2024-11-27T07:25:54,823 INFO [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., pid=91, masterSystemTime=1732692354792 2024-11-27T07:25:54,825 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:25:54,825 INFO [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:25:54,825 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=9cd6e5b712012f5d5b13dc2bcef88d1f, regionState=OPEN, openSeqNum=2, regionLocation=289954c9202d,42117,1732692259671 2024-11-27T07:25:54,827 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 because future has completed 2024-11-27T07:25:54,829 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-27T07:25:54,829 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; OpenRegionProcedure 9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 in 186 msec 2024-11-27T07:25:54,831 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-27T07:25:54,831 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9cd6e5b712012f5d5b13dc2bcef88d1f, ASSIGN in 344 msec 2024-11-27T07:25:54,832 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T07:25:54,832 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732692354832"}]},"ts":"1732692354832"} 2024-11-27T07:25:54,834 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-27T07:25:54,835 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T07:25:54,837 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1860 sec 2024-11-27T07:25:55,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-27T07:25:55,801 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-27T07:25:55,803 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7db9998b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:25:55,803 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 289954c9202d,43057,-1 for getting cluster id 2024-11-27T07:25:55,803 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T07:25:55,805 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '09f43d3f-c175-4713-a9ce-038f5695b74b' 2024-11-27T07:25:55,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T07:25:55,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "09f43d3f-c175-4713-a9ce-038f5695b74b" 2024-11-27T07:25:55,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d9be759, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:25:55,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [289954c9202d,43057,-1] 2024-11-27T07:25:55,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T07:25:55,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:25:55,807 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47898, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T07:25:55,807 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74004f84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:25:55,808 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T07:25:55,809 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=289954c9202d,42117,1732692259671, seqNum=-1] 2024-11-27T07:25:55,809 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T07:25:55,810 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51500, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T07:25:55,811 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-27T07:25:55,812 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 289954c9202d,43057,1732692258941 2024-11-27T07:25:55,812 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@63da3364 2024-11-27T07:25:55,812 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-27T07:25:55,813 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47908, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-27T07:25:55,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-27T07:25:55,815 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T07:25:55,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] procedure2.ProcedureExecutor(1139): Stored pid=92, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-27T07:25:55,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742163_1339 (size=995) 2024-11-27T07:25:56,235 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-27T07:25:56,235 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-27T07:25:56,244 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-27T07:25:56,246 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9cd6e5b712012f5d5b13dc2bcef88d1f, REOPEN/MOVE}] 2024-11-27T07:25:56,247 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9cd6e5b712012f5d5b13dc2bcef88d1f, REOPEN/MOVE 2024-11-27T07:25:56,248 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=9cd6e5b712012f5d5b13dc2bcef88d1f, regionState=CLOSING, regionLocation=289954c9202d,42117,1732692259671 2024-11-27T07:25:56,249 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9cd6e5b712012f5d5b13dc2bcef88d1f, REOPEN/MOVE because future has completed 2024-11-27T07:25:56,249 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T07:25:56,249 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671}] 2024-11-27T07:25:56,401 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(122): Close 9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:25:56,401 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T07:25:56,401 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1722): Closing 9cd6e5b712012f5d5b13dc2bcef88d1f, disabling compactions & flushes 2024-11-27T07:25:56,401 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:25:56,401 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:25:56,401 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. after waiting 0 ms 2024-11-27T07:25:56,401 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:25:56,404 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-27T07:25:56,404 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:25:56,405 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1676): Region close journal for 9cd6e5b712012f5d5b13dc2bcef88d1f: Waiting for close lock at 1732692356401Running coprocessor pre-close hooks at 1732692356401Disabling compacts and flushes for region at 1732692356401Disabling writes for close at 1732692356401Writing region close event to WAL at 1732692356402 (+1 ms)Running coprocessor post-close hooks at 1732692356404 (+2 ms)Closed at 1732692356404 2024-11-27T07:25:56,405 WARN [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegionServer(3299): Not adding moved region record: 9cd6e5b712012f5d5b13dc2bcef88d1f to self. 2024-11-27T07:25:56,406 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(157): Closed 9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:25:56,407 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=9cd6e5b712012f5d5b13dc2bcef88d1f, regionState=CLOSED 2024-11-27T07:25:56,408 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 because future has completed 2024-11-27T07:25:56,410 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-27T07:25:56,410 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; CloseRegionProcedure 9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 in 159 msec 2024-11-27T07:25:56,410 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9cd6e5b712012f5d5b13dc2bcef88d1f, REOPEN/MOVE; state=CLOSED, location=289954c9202d,42117,1732692259671; forceNewPlan=false, retain=true 2024-11-27T07:25:56,561 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=9cd6e5b712012f5d5b13dc2bcef88d1f, regionState=OPENING, regionLocation=289954c9202d,42117,1732692259671 2024-11-27T07:25:56,564 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9cd6e5b712012f5d5b13dc2bcef88d1f, REOPEN/MOVE because future has completed 2024-11-27T07:25:56,565 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671}] 2024-11-27T07:25:56,724 INFO [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:25:56,724 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7752): Opening region: {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} 2024-11-27T07:25:56,725 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:25:56,725 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T07:25:56,725 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7794): checking encryption for 9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:25:56,725 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7797): checking classloading for 9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:25:56,727 INFO [StoreOpener-9cd6e5b712012f5d5b13dc2bcef88d1f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:25:56,728 INFO [StoreOpener-9cd6e5b712012f5d5b13dc2bcef88d1f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9cd6e5b712012f5d5b13dc2bcef88d1f columnFamilyName A 2024-11-27T07:25:56,729 DEBUG [StoreOpener-9cd6e5b712012f5d5b13dc2bcef88d1f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:25:56,729 INFO [StoreOpener-9cd6e5b712012f5d5b13dc2bcef88d1f-1 {}] regionserver.HStore(327): Store=9cd6e5b712012f5d5b13dc2bcef88d1f/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T07:25:56,730 INFO [StoreOpener-9cd6e5b712012f5d5b13dc2bcef88d1f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:25:56,731 INFO [StoreOpener-9cd6e5b712012f5d5b13dc2bcef88d1f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9cd6e5b712012f5d5b13dc2bcef88d1f columnFamilyName B 2024-11-27T07:25:56,731 DEBUG [StoreOpener-9cd6e5b712012f5d5b13dc2bcef88d1f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:25:56,731 INFO [StoreOpener-9cd6e5b712012f5d5b13dc2bcef88d1f-1 {}] regionserver.HStore(327): Store=9cd6e5b712012f5d5b13dc2bcef88d1f/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T07:25:56,731 INFO [StoreOpener-9cd6e5b712012f5d5b13dc2bcef88d1f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:25:56,732 INFO [StoreOpener-9cd6e5b712012f5d5b13dc2bcef88d1f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9cd6e5b712012f5d5b13dc2bcef88d1f columnFamilyName C 2024-11-27T07:25:56,732 DEBUG [StoreOpener-9cd6e5b712012f5d5b13dc2bcef88d1f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:25:56,733 INFO [StoreOpener-9cd6e5b712012f5d5b13dc2bcef88d1f-1 {}] regionserver.HStore(327): Store=9cd6e5b712012f5d5b13dc2bcef88d1f/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T07:25:56,733 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1038): replaying wal for 9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:25:56,734 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:25:56,735 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:25:56,736 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1048): stopping wal replay for 9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:25:56,736 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1060): Cleaning up temporary data for 9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:25:56,736 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T07:25:56,739 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1093): writing seq id for 9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:25:56,740 INFO [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1114): Opened 9cd6e5b712012f5d5b13dc2bcef88d1f; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74331512, jitterRate=0.10762584209442139}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T07:25:56,740 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:25:56,741 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1006): Region open journal for 9cd6e5b712012f5d5b13dc2bcef88d1f: Running coprocessor pre-open hook at 1732692356725Writing region info on filesystem at 1732692356725Initializing all the Stores at 1732692356726 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732692356726Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732692356727 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732692356727Cleaning up temporary data from old regions at 1732692356736 (+9 ms)Running coprocessor post-open hooks at 1732692356740 (+4 ms)Region opened successfully at 1732692356741 (+1 ms) 2024-11-27T07:25:56,742 INFO [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., pid=96, masterSystemTime=1732692356718 2024-11-27T07:25:56,744 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:25:56,744 INFO [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:25:56,744 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=9cd6e5b712012f5d5b13dc2bcef88d1f, regionState=OPEN, openSeqNum=5, regionLocation=289954c9202d,42117,1732692259671 2024-11-27T07:25:56,746 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 because future has completed 2024-11-27T07:25:56,749 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=94 2024-11-27T07:25:56,749 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=94, state=SUCCESS, hasLock=false; OpenRegionProcedure 9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 in 182 msec 2024-11-27T07:25:56,751 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-27T07:25:56,751 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9cd6e5b712012f5d5b13dc2bcef88d1f, REOPEN/MOVE in 503 msec 2024-11-27T07:25:56,754 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-27T07:25:56,754 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 507 msec 2024-11-27T07:25:56,757 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 940 msec 2024-11-27T07:25:56,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-27T07:25:56,759 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-27T07:25:56,760 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f0f1e03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:25:56,760 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 289954c9202d,43057,-1 for getting cluster id 2024-11-27T07:25:56,760 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T07:25:56,761 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '09f43d3f-c175-4713-a9ce-038f5695b74b' 2024-11-27T07:25:56,761 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T07:25:56,761 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "09f43d3f-c175-4713-a9ce-038f5695b74b" 2024-11-27T07:25:56,762 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3335bcf8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:25:56,762 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [289954c9202d,43057,-1] 2024-11-27T07:25:56,762 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T07:25:56,762 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:25:56,763 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47922, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T07:25:56,764 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67f1b641, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:25:56,765 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c3d4f84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:25:56,765 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 289954c9202d,43057,-1 for getting cluster id 2024-11-27T07:25:56,765 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T07:25:56,766 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '09f43d3f-c175-4713-a9ce-038f5695b74b' 2024-11-27T07:25:56,766 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T07:25:56,766 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "09f43d3f-c175-4713-a9ce-038f5695b74b" 2024-11-27T07:25:56,767 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22c29b73, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:25:56,767 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [289954c9202d,43057,-1] 2024-11-27T07:25:56,767 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T07:25:56,767 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:25:56,768 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47936, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T07:25:56,769 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27581969, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:25:56,770 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ad3f2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:25:56,770 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 289954c9202d,43057,-1 for getting cluster id 2024-11-27T07:25:56,770 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T07:25:56,771 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '09f43d3f-c175-4713-a9ce-038f5695b74b' 2024-11-27T07:25:56,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T07:25:56,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "09f43d3f-c175-4713-a9ce-038f5695b74b" 2024-11-27T07:25:56,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c7b9bf5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:25:56,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [289954c9202d,43057,-1] 2024-11-27T07:25:56,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T07:25:56,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:25:56,773 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47946, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T07:25:56,774 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@553ef500, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:25:56,775 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@422fcba0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:25:56,775 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 289954c9202d,43057,-1 for getting cluster id 2024-11-27T07:25:56,775 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T07:25:56,776 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '09f43d3f-c175-4713-a9ce-038f5695b74b' 2024-11-27T07:25:56,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T07:25:56,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "09f43d3f-c175-4713-a9ce-038f5695b74b" 2024-11-27T07:25:56,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64ba434c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:25:56,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [289954c9202d,43057,-1] 2024-11-27T07:25:56,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T07:25:56,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:25:56,777 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47970, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T07:25:56,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4eb36129, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:25:56,779 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72e71a40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:25:56,779 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 289954c9202d,43057,-1 for getting cluster id 2024-11-27T07:25:56,779 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T07:25:56,780 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '09f43d3f-c175-4713-a9ce-038f5695b74b' 2024-11-27T07:25:56,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T07:25:56,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "09f43d3f-c175-4713-a9ce-038f5695b74b" 2024-11-27T07:25:56,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ffbe337, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:25:56,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [289954c9202d,43057,-1] 2024-11-27T07:25:56,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T07:25:56,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:25:56,782 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47978, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T07:25:56,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2af2cb00, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:25:56,783 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61b758c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:25:56,783 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 289954c9202d,43057,-1 for getting cluster id 2024-11-27T07:25:56,783 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T07:25:56,784 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '09f43d3f-c175-4713-a9ce-038f5695b74b' 2024-11-27T07:25:56,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T07:25:56,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "09f43d3f-c175-4713-a9ce-038f5695b74b" 2024-11-27T07:25:56,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b37c877, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:25:56,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [289954c9202d,43057,-1] 2024-11-27T07:25:56,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T07:25:56,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:25:56,786 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47990, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T07:25:56,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f09eed6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:25:56,787 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10e2b5a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:25:56,787 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 289954c9202d,43057,-1 for getting cluster id 2024-11-27T07:25:56,788 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T07:25:56,788 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '09f43d3f-c175-4713-a9ce-038f5695b74b' 2024-11-27T07:25:56,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T07:25:56,789 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "09f43d3f-c175-4713-a9ce-038f5695b74b" 2024-11-27T07:25:56,789 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29d4c822, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:25:56,789 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [289954c9202d,43057,-1] 2024-11-27T07:25:56,789 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T07:25:56,789 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:25:56,790 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48008, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T07:25:56,790 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73c926d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:25:56,791 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d41a701, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:25:56,791 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 289954c9202d,43057,-1 for getting cluster id 2024-11-27T07:25:56,791 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T07:25:56,792 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '09f43d3f-c175-4713-a9ce-038f5695b74b' 2024-11-27T07:25:56,792 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T07:25:56,792 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "09f43d3f-c175-4713-a9ce-038f5695b74b" 2024-11-27T07:25:56,792 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@587a6a40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:25:56,792 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [289954c9202d,43057,-1] 2024-11-27T07:25:56,792 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T07:25:56,793 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:25:56,793 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48032, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T07:25:56,794 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e939af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:25:56,794 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b2f46ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:25:56,794 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 289954c9202d,43057,-1 for getting cluster id 2024-11-27T07:25:56,795 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T07:25:56,795 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '09f43d3f-c175-4713-a9ce-038f5695b74b' 2024-11-27T07:25:56,795 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T07:25:56,796 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "09f43d3f-c175-4713-a9ce-038f5695b74b" 2024-11-27T07:25:56,796 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6047d1f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:25:56,796 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [289954c9202d,43057,-1] 2024-11-27T07:25:56,796 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T07:25:56,796 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:25:56,797 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48044, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T07:25:56,798 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16aed026, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:25:56,798 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@180bd544, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:25:56,798 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 289954c9202d,43057,-1 for getting cluster id 2024-11-27T07:25:56,798 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T07:25:56,799 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '09f43d3f-c175-4713-a9ce-038f5695b74b' 2024-11-27T07:25:56,799 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T07:25:56,799 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "09f43d3f-c175-4713-a9ce-038f5695b74b" 2024-11-27T07:25:56,800 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e092f43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:25:56,800 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [289954c9202d,43057,-1] 2024-11-27T07:25:56,800 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T07:25:56,800 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:25:56,801 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48060, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T07:25:56,801 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4203ca16, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:25:56,802 DEBUG [Thread-1547 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T07:25:56,802 DEBUG [Thread-1562 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T07:25:56,802 DEBUG [Thread-1556 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T07:25:56,802 DEBUG [Thread-1564 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T07:25:56,802 DEBUG [Thread-1560 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T07:25:56,803 DEBUG [Thread-1545 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T07:25:56,803 DEBUG [Thread-1558 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T07:25:56,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=289954c9202d,42117,1732692259671, seqNum=-1] 2024-11-27T07:25:56,803 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=289954c9202d,42117,1732692259671, seqNum=-1] 2024-11-27T07:25:56,803 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=289954c9202d,42117,1732692259671, seqNum=-1] 2024-11-27T07:25:56,803 DEBUG [Thread-1549 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T07:25:56,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T07:25:56,803 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T07:25:56,803 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T07:25:56,803 DEBUG [Thread-1551 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T07:25:56,803 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=289954c9202d,42117,1732692259671, seqNum=-1] 2024-11-27T07:25:56,803 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=289954c9202d,42117,1732692259671, seqNum=-1] 2024-11-27T07:25:56,804 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=289954c9202d,42117,1732692259671, seqNum=-1] 2024-11-27T07:25:56,804 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T07:25:56,804 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T07:25:56,804 DEBUG [Thread-1553 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T07:25:56,804 DEBUG [Thread-1558 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T07:25:56,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=289954c9202d,42117,1732692259671, seqNum=-1] 2024-11-27T07:25:56,804 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=289954c9202d,42117,1732692259671, seqNum=-1] 2024-11-27T07:25:56,804 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51510, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T07:25:56,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T07:25:56,804 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T07:25:56,805 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51508, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T07:25:56,805 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=289954c9202d,42117,1732692259671, seqNum=-1] 2024-11-27T07:25:56,805 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51542, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T07:25:56,806 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51540, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T07:25:56,806 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T07:25:56,806 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51578, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T07:25:56,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=289954c9202d,42117,1732692259671, seqNum=-1] 2024-11-27T07:25:56,806 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51562, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T07:25:56,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T07:25:56,806 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51530, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T07:25:56,807 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51548, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T07:25:56,807 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T07:25:56,807 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51584, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T07:25:56,807 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51586, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T07:25:56,807 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5] 2024-11-27T07:25:56,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5] 2024-11-27T07:25:56,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-27T07:25:56,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-27T07:25:56,809 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T07:25:56,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5] 2024-11-27T07:25:56,809 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5] 2024-11-27T07:25:56,810 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T07:25:56,810 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T07:25:56,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5] 2024-11-27T07:25:56,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5] 2024-11-27T07:25:56,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5] 2024-11-27T07:25:56,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5] 2024-11-27T07:25:56,813 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5] 2024-11-27T07:25:56,814 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5] 2024-11-27T07:25:56,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:25:56,824 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T07:25:56,824 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9cd6e5b712012f5d5b13dc2bcef88d1f 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-27T07:25:56,859 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411277ca0f2c46ba843e2b75bd59c4b516aa7_9cd6e5b712012f5d5b13dc2bcef88d1f is 50, key is test_row_0/A:col10/1732692356822/Put/seqid=0 2024-11-27T07:25:56,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742164_1340 (size=12154) 2024-11-27T07:25:56,867 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:25:56,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:56,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51540 deadline: 1732692416859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:56,872 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411277ca0f2c46ba843e2b75bd59c4b516aa7_9cd6e5b712012f5d5b13dc2bcef88d1f to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411277ca0f2c46ba843e2b75bd59c4b516aa7_9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:25:56,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:56,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:56,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:56,873 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/79bbc270cd324089a506e299686ae483, store: [table=TestAcidGuarantees family=A region=9cd6e5b712012f5d5b13dc2bcef88d1f] 2024-11-27T07:25:56,873 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/79bbc270cd324089a506e299686ae483 is 175, key is test_row_0/A:col10/1732692356822/Put/seqid=0 2024-11-27T07:25:56,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:56,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51586 deadline: 1732692416868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:56,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:56,876 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:56,876 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:56,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51548 deadline: 1732692416868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:56,876 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:56,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:56,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:56,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:56,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:56,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51584 deadline: 1732692416870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:56,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:56,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51510 deadline: 1732692416871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:56,879 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:56,879 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:56,880 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:56,880 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:56,880 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:56,880 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:56,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742165_1341 (size=30955) 2024-11-27T07:25:56,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-27T07:25:56,963 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-27T07:25:56,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:25:56,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. as already flushing 2024-11-27T07:25:56,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:25:56,963 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:56,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:56,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:56,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:56,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51540 deadline: 1732692416979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:56,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:56,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:56,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51586 deadline: 1732692416979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:56,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:56,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:56,984 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:56,984 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:56,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:56,984 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:56,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51548 deadline: 1732692416980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:56,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:56,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:56,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:56,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:56,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51510 deadline: 1732692416989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:56,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:56,993 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:56,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51584 deadline: 1732692416989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:56,993 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:56,993 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:56,993 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:56,993 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:56,993 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:57,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-27T07:25:57,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:25:57,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. as already flushing 2024-11-27T07:25:57,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:25:57,116 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:57,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:57,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:57,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-27T07:25:57,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:57,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51548 deadline: 1732692417190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:57,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:57,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:57,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:57,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:57,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51586 deadline: 1732692417190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:57,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:57,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:57,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:57,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:57,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51540 deadline: 1732692417190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:57,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:57,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:57,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:57,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:57,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51510 deadline: 1732692417199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:57,205 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:57,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:57,205 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:57,205 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:57,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51584 deadline: 1732692417199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:57,205 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:57,205 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:57,205 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:57,269 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-27T07:25:57,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:25:57,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. as already flushing 2024-11-27T07:25:57,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:25:57,269 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:57,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:57,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:57,286 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=18, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/79bbc270cd324089a506e299686ae483 2024-11-27T07:25:57,314 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/B/2755fe75609f42c1bce7d645c75fd5ee is 50, key is test_row_0/B:col10/1732692356822/Put/seqid=0 2024-11-27T07:25:57,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742166_1342 (size=12001) 2024-11-27T07:25:57,421 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-27T07:25:57,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:25:57,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. as already flushing 2024-11-27T07:25:57,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:25:57,422 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:57,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:57,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:57,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-27T07:25:57,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:57,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51548 deadline: 1732692417499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:57,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:57,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:57,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51586 deadline: 1732692417499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:57,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:57,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:57,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:57,502 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:57,502 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:57,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51540 deadline: 1732692417499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:57,502 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:57,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:57,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:57,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:57,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:57,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51510 deadline: 1732692417509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:57,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:57,510 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:57,510 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:57,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51584 deadline: 1732692417509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:57,510 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:57,510 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:57,510 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:57,510 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:57,574 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-27T07:25:57,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:25:57,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. as already flushing 2024-11-27T07:25:57,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:25:57,575 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:57,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:57,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:57,723 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/B/2755fe75609f42c1bce7d645c75fd5ee 2024-11-27T07:25:57,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-27T07:25:57,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:25:57,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. as already flushing 2024-11-27T07:25:57,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:25:57,727 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:57,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:57,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:25:57,743 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/C/3fafbde684bd455d80f2725242a1eb07 is 50, key is test_row_0/C:col10/1732692356822/Put/seqid=0 2024-11-27T07:25:57,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742167_1343 (size=12001) 2024-11-27T07:25:57,748 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/C/3fafbde684bd455d80f2725242a1eb07 2024-11-27T07:25:57,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/79bbc270cd324089a506e299686ae483 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/79bbc270cd324089a506e299686ae483 2024-11-27T07:25:57,754 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/79bbc270cd324089a506e299686ae483, entries=150, sequenceid=18, filesize=30.2 K 2024-11-27T07:25:57,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/B/2755fe75609f42c1bce7d645c75fd5ee as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/2755fe75609f42c1bce7d645c75fd5ee 2024-11-27T07:25:57,760 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/2755fe75609f42c1bce7d645c75fd5ee, entries=150, sequenceid=18, filesize=11.7 K 2024-11-27T07:25:57,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/C/3fafbde684bd455d80f2725242a1eb07 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/3fafbde684bd455d80f2725242a1eb07 2024-11-27T07:25:57,765 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/3fafbde684bd455d80f2725242a1eb07, entries=150, sequenceid=18, filesize=11.7 K 2024-11-27T07:25:57,765 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 9cd6e5b712012f5d5b13dc2bcef88d1f in 941ms, sequenceid=18, compaction requested=false 2024-11-27T07:25:57,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9cd6e5b712012f5d5b13dc2bcef88d1f: 2024-11-27T07:25:57,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-27T07:25:57,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:25:57,886 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 9cd6e5b712012f5d5b13dc2bcef88d1f 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-27T07:25:57,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127ea91402c329d4632838f965788a274ef_9cd6e5b712012f5d5b13dc2bcef88d1f is 50, key is test_row_0/A:col10/1732692356866/Put/seqid=0 2024-11-27T07:25:57,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742168_1344 (size=12154) 2024-11-27T07:25:57,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:25:57,899 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127ea91402c329d4632838f965788a274ef_9cd6e5b712012f5d5b13dc2bcef88d1f to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127ea91402c329d4632838f965788a274ef_9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:25:57,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/7024a54e1de54ada9936496436bb1e3c, store: [table=TestAcidGuarantees family=A region=9cd6e5b712012f5d5b13dc2bcef88d1f] 2024-11-27T07:25:57,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/7024a54e1de54ada9936496436bb1e3c is 175, key is test_row_0/A:col10/1732692356866/Put/seqid=0 2024-11-27T07:25:57,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742169_1345 (size=30955) 2024-11-27T07:25:57,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-27T07:25:58,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:25:58,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. as already flushing 2024-11-27T07:25:58,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:58,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51510 deadline: 1732692418025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:58,033 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:58,033 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:58,033 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:58,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:58,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51584 deadline: 1732692418026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:58,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:58,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:58,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:58,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:58,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51548 deadline: 1732692418032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:58,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:58,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51540 deadline: 1732692418032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:58,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:58,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51586 deadline: 1732692418033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:58,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:58,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:58,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:58,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:58,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:58,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:58,044 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:58,044 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:58,044 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:58,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:58,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51510 deadline: 1732692418141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:58,142 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:58,142 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:58,142 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:58,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:58,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51584 deadline: 1732692418141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:58,147 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:58,147 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:58,147 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:58,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:58,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51548 deadline: 1732692418149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:58,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:58,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:58,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:58,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:58,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51540 deadline: 1732692418149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:58,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:58,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51586 deadline: 1732692418149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:58,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:58,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:58,156 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:58,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:58,156 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:58,156 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:58,312 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/7024a54e1de54ada9936496436bb1e3c 2024-11-27T07:25:58,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/B/fe1a11116b944dc6b8440584cf8bd1d3 is 50, key is test_row_0/B:col10/1732692356866/Put/seqid=0 2024-11-27T07:25:58,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742170_1346 (size=12001) 2024-11-27T07:25:58,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:58,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51510 deadline: 1732692418349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:58,356 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:58,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:58,356 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:58,356 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:58,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51584 deadline: 1732692418349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:58,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:58,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:58,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:58,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:58,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51548 deadline: 1732692418359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:58,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:58,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51586 deadline: 1732692418359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:58,368 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:58,368 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:58,368 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:58,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:58,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:58,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:58,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:58,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51540 deadline: 1732692418369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:58,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:58,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:58,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:58,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:58,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51584 deadline: 1732692418670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:58,674 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:58,674 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:58,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:58,674 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:58,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51510 deadline: 1732692418670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:58,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:58,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:58,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:58,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:58,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51586 deadline: 1732692418679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:58,686 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:58,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:58,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:58,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:58,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51548 deadline: 1732692418679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:58,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:58,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:58,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:58,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:58,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51540 deadline: 1732692418689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:58,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:58,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:58,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:58,722 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/B/fe1a11116b944dc6b8440584cf8bd1d3 2024-11-27T07:25:58,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/C/b0a1adabfb6841bdaaf4d2a4581f0040 is 50, key is test_row_0/C:col10/1732692356866/Put/seqid=0 2024-11-27T07:25:58,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742171_1347 (size=12001) 2024-11-27T07:25:58,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-27T07:25:59,054 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T07:25:59,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:59,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51584 deadline: 1732692419180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:59,187 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:59,187 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:59,187 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:59,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:59,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51510 deadline: 1732692419180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:59,189 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:59,189 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:59,189 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:59,192 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/C/b0a1adabfb6841bdaaf4d2a4581f0040 2024-11-27T07:25:59,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:59,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51586 deadline: 1732692419189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:59,199 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:59,199 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:59,199 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:59,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:59,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51548 deadline: 1732692419199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:59,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:59,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:59,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:59,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:25:59,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51540 deadline: 1732692419209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:25:59,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:59,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:25:59,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:25:59,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/7024a54e1de54ada9936496436bb1e3c as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/7024a54e1de54ada9936496436bb1e3c 2024-11-27T07:25:59,228 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/7024a54e1de54ada9936496436bb1e3c, entries=150, sequenceid=40, filesize=30.2 K 2024-11-27T07:25:59,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/B/fe1a11116b944dc6b8440584cf8bd1d3 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/fe1a11116b944dc6b8440584cf8bd1d3 2024-11-27T07:25:59,235 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/fe1a11116b944dc6b8440584cf8bd1d3, entries=150, sequenceid=40, filesize=11.7 K 2024-11-27T07:25:59,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/C/b0a1adabfb6841bdaaf4d2a4581f0040 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/b0a1adabfb6841bdaaf4d2a4581f0040 2024-11-27T07:25:59,242 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/b0a1adabfb6841bdaaf4d2a4581f0040, entries=150, sequenceid=40, filesize=11.7 K 2024-11-27T07:25:59,244 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 9cd6e5b712012f5d5b13dc2bcef88d1f in 1358ms, sequenceid=40, compaction requested=false 2024-11-27T07:25:59,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 9cd6e5b712012f5d5b13dc2bcef88d1f: 2024-11-27T07:25:59,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:25:59,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-27T07:25:59,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-27T07:25:59,249 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-27T07:25:59,249 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4360 sec 2024-11-27T07:25:59,251 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 2.4430 sec 2024-11-27T07:26:00,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:26:00,202 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T07:26:00,202 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9cd6e5b712012f5d5b13dc2bcef88d1f 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-27T07:26:00,213 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411279328e4f9584c4754a2b73dc2ed8392ee_9cd6e5b712012f5d5b13dc2bcef88d1f is 50, key is test_row_0/A:col10/1732692360202/Put/seqid=0 2024-11-27T07:26:00,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742172_1348 (size=17034) 2024-11-27T07:26:00,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:00,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51548 deadline: 1732692420267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:00,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:00,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:00,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:00,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:00,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51540 deadline: 1732692420269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:00,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:00,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:00,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:00,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:00,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51584 deadline: 1732692420276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:00,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:00,289 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:00,289 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:00,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51586 deadline: 1732692420278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:00,289 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:00,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:00,289 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:00,289 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:00,289 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:00,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51510 deadline: 1732692420278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:00,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:00,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:00,290 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:00,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:00,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51540 deadline: 1732692420389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:00,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:00,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:00,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:00,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:00,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51548 deadline: 1732692420390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:00,397 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:00,397 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:00,398 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:00,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:00,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51510 deadline: 1732692420399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:00,408 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:00,408 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:00,408 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:00,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:00,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51584 deadline: 1732692420399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:00,409 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:00,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:00,409 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:00,409 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:00,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51586 deadline: 1732692420400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:00,409 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:00,410 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:00,410 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:00,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:00,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51540 deadline: 1732692420599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:00,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:00,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:00,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:00,617 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:26:00,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:00,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51548 deadline: 1732692420609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:00,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:00,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:00,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:00,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:00,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51510 deadline: 1732692420620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:00,628 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:00,628 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:00,628 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:00,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:00,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51584 deadline: 1732692420620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:00,629 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:00,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:00,629 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:00,629 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:00,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51586 deadline: 1732692420620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:00,629 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:00,629 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:00,629 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:00,631 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411279328e4f9584c4754a2b73dc2ed8392ee_9cd6e5b712012f5d5b13dc2bcef88d1f to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411279328e4f9584c4754a2b73dc2ed8392ee_9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:26:00,633 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/60d9b84bdaa04f939dc621f0178a3111, store: [table=TestAcidGuarantees family=A region=9cd6e5b712012f5d5b13dc2bcef88d1f] 2024-11-27T07:26:00,633 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/60d9b84bdaa04f939dc621f0178a3111 is 175, key is test_row_0/A:col10/1732692360202/Put/seqid=0 2024-11-27T07:26:00,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742173_1349 (size=48139) 2024-11-27T07:26:00,655 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=56, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/60d9b84bdaa04f939dc621f0178a3111 2024-11-27T07:26:00,672 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/B/a538e2170a3644a3ae706a5369dfceb6 is 50, key is test_row_0/B:col10/1732692360202/Put/seqid=0 2024-11-27T07:26:00,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742174_1350 (size=12001) 2024-11-27T07:26:00,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:00,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51540 deadline: 1732692420909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:00,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:00,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:00,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:00,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:00,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51548 deadline: 1732692420930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:00,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:00,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:00,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:00,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:00,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51510 deadline: 1732692420939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:00,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:00,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51584 deadline: 1732692420939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:00,948 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:00,948 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:00,948 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:00,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:00,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51586 deadline: 1732692420939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:00,948 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:00,948 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:00,948 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:00,949 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:00,949 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:00,949 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:00,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-27T07:26:00,969 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T07:26:00,969 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T07:26:00,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-27T07:26:00,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-27T07:26:00,972 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T07:26:00,972 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T07:26:00,972 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T07:26:01,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-27T07:26:01,115 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/B/a538e2170a3644a3ae706a5369dfceb6 2024-11-27T07:26:01,126 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-27T07:26:01,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:01,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. as already flushing 2024-11-27T07:26:01,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:01,127 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:01,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:01,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:01,142 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/C/e282058b049a460b83fa3ec126537667 is 50, key is test_row_0/C:col10/1732692360202/Put/seqid=0 2024-11-27T07:26:01,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742175_1351 (size=12001) 2024-11-27T07:26:01,280 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-27T07:26:01,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:01,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. as already flushing 2024-11-27T07:26:01,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:01,280 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:01,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:01,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:01,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-27T07:26:01,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-27T07:26:01,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:01,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. as already flushing 2024-11-27T07:26:01,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:01,435 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:01,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:01,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:01,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:01,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51540 deadline: 1732692421429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:01,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:01,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:01,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:01,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:01,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51548 deadline: 1732692421449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:01,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:01,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:01,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:01,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:01,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51510 deadline: 1732692421459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:01,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:01,468 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:01,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:01,468 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:01,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51586 deadline: 1732692421459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:01,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:01,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51584 deadline: 1732692421459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:01,468 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:01,468 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:01,468 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:01,468 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:01,468 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:01,468 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:01,586 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/C/e282058b049a460b83fa3ec126537667 2024-11-27T07:26:01,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-27T07:26:01,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:01,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. as already flushing 2024-11-27T07:26:01,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:01,587 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:01,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:01,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:01,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/60d9b84bdaa04f939dc621f0178a3111 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/60d9b84bdaa04f939dc621f0178a3111 2024-11-27T07:26:01,594 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/60d9b84bdaa04f939dc621f0178a3111, entries=250, sequenceid=56, filesize=47.0 K 2024-11-27T07:26:01,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/B/a538e2170a3644a3ae706a5369dfceb6 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/a538e2170a3644a3ae706a5369dfceb6 2024-11-27T07:26:01,598 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/a538e2170a3644a3ae706a5369dfceb6, entries=150, sequenceid=56, filesize=11.7 K 2024-11-27T07:26:01,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/C/e282058b049a460b83fa3ec126537667 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/e282058b049a460b83fa3ec126537667 2024-11-27T07:26:01,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-27T07:26:01,602 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/e282058b049a460b83fa3ec126537667, entries=150, sequenceid=56, filesize=11.7 K 2024-11-27T07:26:01,603 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for 9cd6e5b712012f5d5b13dc2bcef88d1f in 1401ms, sequenceid=56, compaction requested=true 2024-11-27T07:26:01,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9cd6e5b712012f5d5b13dc2bcef88d1f: 2024-11-27T07:26:01,603 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:26:01,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cd6e5b712012f5d5b13dc2bcef88d1f:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T07:26:01,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:26:01,604 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:26:01,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cd6e5b712012f5d5b13dc2bcef88d1f:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T07:26:01,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:26:01,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cd6e5b712012f5d5b13dc2bcef88d1f:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T07:26:01,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:26:01,604 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:26:01,604 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1541): 9cd6e5b712012f5d5b13dc2bcef88d1f/A is initiating minor compaction (all files) 2024-11-27T07:26:01,604 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cd6e5b712012f5d5b13dc2bcef88d1f/A in TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:01,605 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/79bbc270cd324089a506e299686ae483, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/7024a54e1de54ada9936496436bb1e3c, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/60d9b84bdaa04f939dc621f0178a3111] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp, totalSize=107.5 K 2024-11-27T07:26:01,605 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:01,605 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. files: [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/79bbc270cd324089a506e299686ae483, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/7024a54e1de54ada9936496436bb1e3c, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/60d9b84bdaa04f939dc621f0178a3111] 2024-11-27T07:26:01,605 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:26:01,605 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 9cd6e5b712012f5d5b13dc2bcef88d1f/B is initiating minor compaction (all files) 2024-11-27T07:26:01,605 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cd6e5b712012f5d5b13dc2bcef88d1f/B in TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:01,605 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79bbc270cd324089a506e299686ae483, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=18, earliestPutTs=1732692356822 2024-11-27T07:26:01,605 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/2755fe75609f42c1bce7d645c75fd5ee, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/fe1a11116b944dc6b8440584cf8bd1d3, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/a538e2170a3644a3ae706a5369dfceb6] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp, totalSize=35.2 K 2024-11-27T07:26:01,605 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7024a54e1de54ada9936496436bb1e3c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732692356866 2024-11-27T07:26:01,605 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 2755fe75609f42c1bce7d645c75fd5ee, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=18, earliestPutTs=1732692356822 2024-11-27T07:26:01,605 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60d9b84bdaa04f939dc621f0178a3111, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1732692358025 2024-11-27T07:26:01,608 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting fe1a11116b944dc6b8440584cf8bd1d3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732692356866 2024-11-27T07:26:01,608 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting a538e2170a3644a3ae706a5369dfceb6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1732692358025 2024-11-27T07:26:01,609 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9cd6e5b712012f5d5b13dc2bcef88d1f] 2024-11-27T07:26:01,611 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411275922efc6dc9e42aab43d06b67fcd7764_9cd6e5b712012f5d5b13dc2bcef88d1f store=[table=TestAcidGuarantees family=A region=9cd6e5b712012f5d5b13dc2bcef88d1f] 2024-11-27T07:26:01,613 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411275922efc6dc9e42aab43d06b67fcd7764_9cd6e5b712012f5d5b13dc2bcef88d1f, store=[table=TestAcidGuarantees family=A region=9cd6e5b712012f5d5b13dc2bcef88d1f] 2024-11-27T07:26:01,614 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411275922efc6dc9e42aab43d06b67fcd7764_9cd6e5b712012f5d5b13dc2bcef88d1f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9cd6e5b712012f5d5b13dc2bcef88d1f] 2024-11-27T07:26:01,622 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cd6e5b712012f5d5b13dc2bcef88d1f#B#compaction#298 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:26:01,622 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/B/a0ba446e16694485b6d70e0500c533eb is 50, key is test_row_0/B:col10/1732692360202/Put/seqid=0 2024-11-27T07:26:01,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742176_1352 (size=4469) 2024-11-27T07:26:01,625 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cd6e5b712012f5d5b13dc2bcef88d1f#A#compaction#297 average throughput is 1.63 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:26:01,625 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/db34162c92a84b46bace782fe7821190 is 175, key is test_row_0/A:col10/1732692360202/Put/seqid=0 2024-11-27T07:26:01,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742177_1353 (size=12104) 2024-11-27T07:26:01,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742178_1354 (size=31058) 2024-11-27T07:26:01,744 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-27T07:26:01,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:01,745 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 9cd6e5b712012f5d5b13dc2bcef88d1f 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-27T07:26:01,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112742a651d45e7e4fdaa3108d66e77c7bd4_9cd6e5b712012f5d5b13dc2bcef88d1f is 50, key is test_row_0/A:col10/1732692360276/Put/seqid=0 2024-11-27T07:26:01,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742179_1355 (size=12154) 2024-11-27T07:26:02,032 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/B/a0ba446e16694485b6d70e0500c533eb as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/a0ba446e16694485b6d70e0500c533eb 2024-11-27T07:26:02,036 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9cd6e5b712012f5d5b13dc2bcef88d1f/B of 9cd6e5b712012f5d5b13dc2bcef88d1f into a0ba446e16694485b6d70e0500c533eb(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:26:02,036 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cd6e5b712012f5d5b13dc2bcef88d1f: 2024-11-27T07:26:02,036 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., storeName=9cd6e5b712012f5d5b13dc2bcef88d1f/B, priority=13, startTime=1732692361604; duration=0sec 2024-11-27T07:26:02,036 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:26:02,036 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cd6e5b712012f5d5b13dc2bcef88d1f:B 2024-11-27T07:26:02,036 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:26:02,038 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:26:02,038 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 9cd6e5b712012f5d5b13dc2bcef88d1f/C is initiating minor compaction (all files) 2024-11-27T07:26:02,038 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cd6e5b712012f5d5b13dc2bcef88d1f/C in TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:02,038 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/3fafbde684bd455d80f2725242a1eb07, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/b0a1adabfb6841bdaaf4d2a4581f0040, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/e282058b049a460b83fa3ec126537667] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp, totalSize=35.2 K 2024-11-27T07:26:02,039 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/db34162c92a84b46bace782fe7821190 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/db34162c92a84b46bace782fe7821190 2024-11-27T07:26:02,039 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 3fafbde684bd455d80f2725242a1eb07, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=18, earliestPutTs=1732692356822 2024-11-27T07:26:02,039 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting b0a1adabfb6841bdaaf4d2a4581f0040, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732692356866 2024-11-27T07:26:02,039 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting e282058b049a460b83fa3ec126537667, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1732692358025 2024-11-27T07:26:02,044 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9cd6e5b712012f5d5b13dc2bcef88d1f/A of 9cd6e5b712012f5d5b13dc2bcef88d1f into db34162c92a84b46bace782fe7821190(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:26:02,044 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cd6e5b712012f5d5b13dc2bcef88d1f: 2024-11-27T07:26:02,044 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., storeName=9cd6e5b712012f5d5b13dc2bcef88d1f/A, priority=13, startTime=1732692361603; duration=0sec 2024-11-27T07:26:02,044 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:26:02,044 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cd6e5b712012f5d5b13dc2bcef88d1f:A 2024-11-27T07:26:02,050 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cd6e5b712012f5d5b13dc2bcef88d1f#C#compaction#300 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:26:02,050 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/C/498477fe7a9548daa93c462a2f24b842 is 50, key is test_row_0/C:col10/1732692360202/Put/seqid=0 2024-11-27T07:26:02,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742180_1356 (size=12104) 2024-11-27T07:26:02,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-27T07:26:02,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:26:02,158 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112742a651d45e7e4fdaa3108d66e77c7bd4_9cd6e5b712012f5d5b13dc2bcef88d1f to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112742a651d45e7e4fdaa3108d66e77c7bd4_9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:26:02,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/6068ebc37dd64ae0a7d83c0a19f239ac, store: [table=TestAcidGuarantees family=A region=9cd6e5b712012f5d5b13dc2bcef88d1f] 2024-11-27T07:26:02,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/6068ebc37dd64ae0a7d83c0a19f239ac is 175, key is test_row_0/A:col10/1732692360276/Put/seqid=0 2024-11-27T07:26:02,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742181_1357 (size=30955) 2024-11-27T07:26:02,170 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=77, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/6068ebc37dd64ae0a7d83c0a19f239ac 2024-11-27T07:26:02,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/B/df66a6e50cda4480bb0f9833ef900020 is 50, key is test_row_0/B:col10/1732692360276/Put/seqid=0 2024-11-27T07:26:02,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742182_1358 (size=12001) 2024-11-27T07:26:02,194 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/B/df66a6e50cda4480bb0f9833ef900020 2024-11-27T07:26:02,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/C/1b65b08c5b0c4d63af85f743429f075a is 50, key is test_row_0/C:col10/1732692360276/Put/seqid=0 2024-11-27T07:26:02,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742183_1359 (size=12001) 2024-11-27T07:26:02,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:26:02,454 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. as already flushing 2024-11-27T07:26:02,462 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/C/498477fe7a9548daa93c462a2f24b842 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/498477fe7a9548daa93c462a2f24b842 2024-11-27T07:26:02,469 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9cd6e5b712012f5d5b13dc2bcef88d1f/C of 9cd6e5b712012f5d5b13dc2bcef88d1f into 498477fe7a9548daa93c462a2f24b842(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:26:02,469 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cd6e5b712012f5d5b13dc2bcef88d1f: 2024-11-27T07:26:02,469 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., storeName=9cd6e5b712012f5d5b13dc2bcef88d1f/C, priority=13, startTime=1732692361604; duration=0sec 2024-11-27T07:26:02,469 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:26:02,469 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cd6e5b712012f5d5b13dc2bcef88d1f:C 2024-11-27T07:26:02,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:02,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51510 deadline: 1732692422479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:02,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:02,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51548 deadline: 1732692422483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:02,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:02,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:02,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:02,489 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 289954c9202d,42117,1732692259671 for 'test_row_2' in 9cd6e5b712012f5d5b13dc2bcef88d1f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T07:26:02,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:02,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:02,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:02,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:02,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51540 deadline: 1732692422484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:02,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:02,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:02,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:02,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:02,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51584 deadline: 1732692422488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:02,492 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:02,492 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:02,492 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:02,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:02,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51586 deadline: 1732692422489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:02,498 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:02,498 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:02,498 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:02,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:02,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51548 deadline: 1732692422599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:02,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:02,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51540 deadline: 1732692422599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:02,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:02,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:02,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51584 deadline: 1732692422600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:02,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:02,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:02,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:02,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:02,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:02,607 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:02,607 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:02,607 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:02,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:02,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51586 deadline: 1732692422609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:02,616 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:02,616 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:02,616 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:02,636 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/C/1b65b08c5b0c4d63af85f743429f075a 2024-11-27T07:26:02,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/6068ebc37dd64ae0a7d83c0a19f239ac as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/6068ebc37dd64ae0a7d83c0a19f239ac 2024-11-27T07:26:02,646 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/6068ebc37dd64ae0a7d83c0a19f239ac, entries=150, sequenceid=77, filesize=30.2 K 2024-11-27T07:26:02,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/B/df66a6e50cda4480bb0f9833ef900020 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/df66a6e50cda4480bb0f9833ef900020 2024-11-27T07:26:02,650 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/df66a6e50cda4480bb0f9833ef900020, entries=150, sequenceid=77, filesize=11.7 K 2024-11-27T07:26:02,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/C/1b65b08c5b0c4d63af85f743429f075a as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/1b65b08c5b0c4d63af85f743429f075a 2024-11-27T07:26:02,657 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/1b65b08c5b0c4d63af85f743429f075a, entries=150, sequenceid=77, filesize=11.7 K 2024-11-27T07:26:02,658 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 9cd6e5b712012f5d5b13dc2bcef88d1f in 913ms, sequenceid=77, compaction requested=false 2024-11-27T07:26:02,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 9cd6e5b712012f5d5b13dc2bcef88d1f: 2024-11-27T07:26:02,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:02,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-27T07:26:02,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-27T07:26:02,665 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-27T07:26:02,666 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6880 sec 2024-11-27T07:26:02,671 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.6960 sec 2024-11-27T07:26:02,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:26:02,810 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T07:26:02,810 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9cd6e5b712012f5d5b13dc2bcef88d1f 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-27T07:26:02,817 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411273cdc2fa0bd29466399a132da6a7dbda8_9cd6e5b712012f5d5b13dc2bcef88d1f is 50, key is test_row_0/A:col10/1732692362809/Put/seqid=0 2024-11-27T07:26:02,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742184_1360 (size=14594) 2024-11-27T07:26:02,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:02,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51540 deadline: 1732692422860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:02,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:02,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:02,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:02,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:02,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51548 deadline: 1732692422865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:02,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:02,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:02,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:02,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:02,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51584 deadline: 1732692422866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:02,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:02,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51586 deadline: 1732692422867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:02,878 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:02,878 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:02,878 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:02,880 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:02,880 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:02,880 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:02,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:02,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51548 deadline: 1732692422979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:02,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:02,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:02,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:02,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:02,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51584 deadline: 1732692422979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:02,985 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:02,985 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:02,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:02,985 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:02,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51540 deadline: 1732692422981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:02,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:02,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:02,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:02,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:02,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51586 deadline: 1732692422989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:02,996 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:02,996 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:02,996 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:03,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-27T07:26:03,119 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T07:26:03,120 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T07:26:03,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-27T07:26:03,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-27T07:26:03,121 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T07:26:03,122 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T07:26:03,122 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T07:26:03,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:03,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51548 deadline: 1732692423189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:03,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:03,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:03,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:03,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:03,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51540 deadline: 1732692423189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:03,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:03,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:03,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:03,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:03,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51584 deadline: 1732692423189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:03,197 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:03,197 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:03,197 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:03,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:03,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51586 deadline: 1732692423199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:03,206 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:03,206 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:03,206 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:03,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-27T07:26:03,241 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:26:03,244 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411273cdc2fa0bd29466399a132da6a7dbda8_9cd6e5b712012f5d5b13dc2bcef88d1f to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411273cdc2fa0bd29466399a132da6a7dbda8_9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:26:03,245 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/431d956281d84a719f74a1c4da5c0b87, store: [table=TestAcidGuarantees family=A region=9cd6e5b712012f5d5b13dc2bcef88d1f] 2024-11-27T07:26:03,246 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/431d956281d84a719f74a1c4da5c0b87 is 175, key is test_row_0/A:col10/1732692362809/Put/seqid=0 2024-11-27T07:26:03,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742185_1361 (size=39549) 2024-11-27T07:26:03,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-27T07:26:03,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:03,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. as already flushing 2024-11-27T07:26:03,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:03,274 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:03,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:03,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:03,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-27T07:26:03,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:03,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. as already flushing 2024-11-27T07:26:03,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:03,426 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:03,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:03,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:03,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-27T07:26:03,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:03,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51548 deadline: 1732692423499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:03,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:03,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:03,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:03,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:03,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51540 deadline: 1732692423499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:03,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:03,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:03,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:03,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:03,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51584 deadline: 1732692423509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:03,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:03,514 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:03,514 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:03,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51586 deadline: 1732692423509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:03,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:03,515 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:03,515 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:03,515 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:03,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-27T07:26:03,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:03,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. as already flushing 2024-11-27T07:26:03,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:03,579 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:03,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:03,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:03,658 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=96, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/431d956281d84a719f74a1c4da5c0b87 2024-11-27T07:26:03,664 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/B/6a3361f5a3b44c568bfdc171b42326e0 is 50, key is test_row_0/B:col10/1732692362809/Put/seqid=0 2024-11-27T07:26:03,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742186_1362 (size=12001) 2024-11-27T07:26:03,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-27T07:26:03,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:03,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. as already flushing 2024-11-27T07:26:03,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:03,732 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:03,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:03,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:03,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-27T07:26:03,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-27T07:26:03,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:03,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. as already flushing 2024-11-27T07:26:03,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:03,884 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:03,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:03,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:04,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:04,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51548 deadline: 1732692424009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:04,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:04,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:04,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:04,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:04,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51540 deadline: 1732692424009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:04,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:04,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:04,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:04,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:04,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51584 deadline: 1732692424019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:04,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:04,026 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:04,026 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:04,026 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:04,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51586 deadline: 1732692424019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:04,027 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:04,027 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:04,027 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:04,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-27T07:26:04,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:04,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. as already flushing 2024-11-27T07:26:04,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:04,037 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:04,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:04,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:04,068 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/B/6a3361f5a3b44c568bfdc171b42326e0 2024-11-27T07:26:04,075 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/C/0e378feb740c4c878a496a4d488e4bc5 is 50, key is test_row_0/C:col10/1732692362809/Put/seqid=0 2024-11-27T07:26:04,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742187_1363 (size=12001) 2024-11-27T07:26:04,190 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-27T07:26:04,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:04,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. as already flushing 2024-11-27T07:26:04,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:04,190 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:04,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:04,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:04,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-27T07:26:04,343 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-27T07:26:04,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:04,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. as already flushing 2024-11-27T07:26:04,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:04,343 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:04,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:04,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:04,478 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/C/0e378feb740c4c878a496a4d488e4bc5 2024-11-27T07:26:04,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/431d956281d84a719f74a1c4da5c0b87 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/431d956281d84a719f74a1c4da5c0b87 2024-11-27T07:26:04,486 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/431d956281d84a719f74a1c4da5c0b87, entries=200, sequenceid=96, filesize=38.6 K 2024-11-27T07:26:04,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/B/6a3361f5a3b44c568bfdc171b42326e0 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/6a3361f5a3b44c568bfdc171b42326e0 2024-11-27T07:26:04,490 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/6a3361f5a3b44c568bfdc171b42326e0, entries=150, sequenceid=96, filesize=11.7 K 2024-11-27T07:26:04,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/C/0e378feb740c4c878a496a4d488e4bc5 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/0e378feb740c4c878a496a4d488e4bc5 2024-11-27T07:26:04,494 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/0e378feb740c4c878a496a4d488e4bc5, entries=150, sequenceid=96, filesize=11.7 K 2024-11-27T07:26:04,495 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 9cd6e5b712012f5d5b13dc2bcef88d1f in 1685ms, sequenceid=96, compaction requested=true 2024-11-27T07:26:04,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9cd6e5b712012f5d5b13dc2bcef88d1f: 2024-11-27T07:26:04,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cd6e5b712012f5d5b13dc2bcef88d1f:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T07:26:04,495 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:26:04,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:26:04,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cd6e5b712012f5d5b13dc2bcef88d1f:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T07:26:04,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:26:04,495 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:26:04,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cd6e5b712012f5d5b13dc2bcef88d1f:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T07:26:04,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:26:04,495 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-27T07:26:04,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:04,496 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 9cd6e5b712012f5d5b13dc2bcef88d1f 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-27T07:26:04,498 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:26:04,498 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1541): 9cd6e5b712012f5d5b13dc2bcef88d1f/A is initiating minor compaction (all files) 2024-11-27T07:26:04,498 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cd6e5b712012f5d5b13dc2bcef88d1f/A in TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:04,499 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/db34162c92a84b46bace782fe7821190, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/6068ebc37dd64ae0a7d83c0a19f239ac, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/431d956281d84a719f74a1c4da5c0b87] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp, totalSize=99.2 K 2024-11-27T07:26:04,499 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:04,499 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. files: [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/db34162c92a84b46bace782fe7821190, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/6068ebc37dd64ae0a7d83c0a19f239ac, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/431d956281d84a719f74a1c4da5c0b87] 2024-11-27T07:26:04,499 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting db34162c92a84b46bace782fe7821190, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1732692358025 2024-11-27T07:26:04,499 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:26:04,499 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 9cd6e5b712012f5d5b13dc2bcef88d1f/B is initiating minor compaction (all files) 2024-11-27T07:26:04,499 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cd6e5b712012f5d5b13dc2bcef88d1f/B in TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:04,499 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6068ebc37dd64ae0a7d83c0a19f239ac, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732692360263 2024-11-27T07:26:04,499 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/a0ba446e16694485b6d70e0500c533eb, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/df66a6e50cda4480bb0f9833ef900020, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/6a3361f5a3b44c568bfdc171b42326e0] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp, totalSize=35.3 K 2024-11-27T07:26:04,501 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 431d956281d84a719f74a1c4da5c0b87, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732692362477 2024-11-27T07:26:04,501 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting a0ba446e16694485b6d70e0500c533eb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1732692358025 2024-11-27T07:26:04,501 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting df66a6e50cda4480bb0f9833ef900020, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732692360263 2024-11-27T07:26:04,502 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a3361f5a3b44c568bfdc171b42326e0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732692362486 2024-11-27T07:26:04,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:26:04,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. as already flushing 2024-11-27T07:26:04,514 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9cd6e5b712012f5d5b13dc2bcef88d1f] 2024-11-27T07:26:04,515 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cd6e5b712012f5d5b13dc2bcef88d1f#B#compaction#306 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:26:04,515 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/B/55689899011c4000a68c05ce335dbaa3 is 50, key is test_row_0/B:col10/1732692362809/Put/seqid=0 2024-11-27T07:26:04,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112717c200f61b98477983234c3cfe89163d_9cd6e5b712012f5d5b13dc2bcef88d1f is 50, key is test_row_0/A:col10/1732692362865/Put/seqid=0 2024-11-27T07:26:04,526 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127a0bf341b182f42798c6d46a6824276d7_9cd6e5b712012f5d5b13dc2bcef88d1f store=[table=TestAcidGuarantees family=A region=9cd6e5b712012f5d5b13dc2bcef88d1f] 2024-11-27T07:26:04,528 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127a0bf341b182f42798c6d46a6824276d7_9cd6e5b712012f5d5b13dc2bcef88d1f, store=[table=TestAcidGuarantees family=A region=9cd6e5b712012f5d5b13dc2bcef88d1f] 2024-11-27T07:26:04,528 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127a0bf341b182f42798c6d46a6824276d7_9cd6e5b712012f5d5b13dc2bcef88d1f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9cd6e5b712012f5d5b13dc2bcef88d1f] 2024-11-27T07:26:04,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742188_1364 (size=12207) 2024-11-27T07:26:04,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742189_1365 (size=12154) 2024-11-27T07:26:04,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742190_1366 (size=4469) 2024-11-27T07:26:04,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:04,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51510 deadline: 1732692424592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:04,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:04,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:04,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:04,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:04,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51510 deadline: 1732692424699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:04,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:04,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:04,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:04,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:04,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51510 deadline: 1732692424919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:04,932 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:04,932 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:04,932 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:04,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:26:04,941 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cd6e5b712012f5d5b13dc2bcef88d1f#A#compaction#307 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:26:04,941 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/B/55689899011c4000a68c05ce335dbaa3 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/55689899011c4000a68c05ce335dbaa3 2024-11-27T07:26:04,942 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/abd923c0d5ac4a0e98edff57a47689f9 is 175, key is test_row_0/A:col10/1732692362809/Put/seqid=0 2024-11-27T07:26:04,943 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112717c200f61b98477983234c3cfe89163d_9cd6e5b712012f5d5b13dc2bcef88d1f to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112717c200f61b98477983234c3cfe89163d_9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:26:04,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/6a6a3fd18132467ea5a621412a82cfba, store: [table=TestAcidGuarantees family=A region=9cd6e5b712012f5d5b13dc2bcef88d1f] 2024-11-27T07:26:04,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/6a6a3fd18132467ea5a621412a82cfba is 175, key is test_row_0/A:col10/1732692362865/Put/seqid=0 2024-11-27T07:26:04,954 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9cd6e5b712012f5d5b13dc2bcef88d1f/B of 9cd6e5b712012f5d5b13dc2bcef88d1f into 55689899011c4000a68c05ce335dbaa3(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:26:04,954 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cd6e5b712012f5d5b13dc2bcef88d1f: 2024-11-27T07:26:04,954 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., storeName=9cd6e5b712012f5d5b13dc2bcef88d1f/B, priority=13, startTime=1732692364495; duration=0sec 2024-11-27T07:26:04,954 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:26:04,954 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cd6e5b712012f5d5b13dc2bcef88d1f:B 2024-11-27T07:26:04,954 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:26:04,958 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:26:04,958 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 9cd6e5b712012f5d5b13dc2bcef88d1f/C is initiating minor compaction (all files) 2024-11-27T07:26:04,958 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cd6e5b712012f5d5b13dc2bcef88d1f/C in TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:04,958 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/498477fe7a9548daa93c462a2f24b842, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/1b65b08c5b0c4d63af85f743429f075a, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/0e378feb740c4c878a496a4d488e4bc5] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp, totalSize=35.3 K 2024-11-27T07:26:04,960 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 498477fe7a9548daa93c462a2f24b842, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1732692358025 2024-11-27T07:26:04,960 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b65b08c5b0c4d63af85f743429f075a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732692360263 2024-11-27T07:26:04,960 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e378feb740c4c878a496a4d488e4bc5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732692362486 2024-11-27T07:26:04,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742192_1368 (size=30955) 2024-11-27T07:26:04,970 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=116, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/6a6a3fd18132467ea5a621412a82cfba 2024-11-27T07:26:04,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742191_1367 (size=31161) 2024-11-27T07:26:04,990 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cd6e5b712012f5d5b13dc2bcef88d1f#C#compaction#309 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:26:04,991 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/C/6b0dece1caba4c7bb71ae164750bce10 is 50, key is test_row_0/C:col10/1732692362809/Put/seqid=0 2024-11-27T07:26:04,993 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/abd923c0d5ac4a0e98edff57a47689f9 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/abd923c0d5ac4a0e98edff57a47689f9 2024-11-27T07:26:04,999 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9cd6e5b712012f5d5b13dc2bcef88d1f/A of 9cd6e5b712012f5d5b13dc2bcef88d1f into abd923c0d5ac4a0e98edff57a47689f9(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:26:05,000 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cd6e5b712012f5d5b13dc2bcef88d1f: 2024-11-27T07:26:05,000 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., storeName=9cd6e5b712012f5d5b13dc2bcef88d1f/A, priority=13, startTime=1732692364495; duration=0sec 2024-11-27T07:26:05,000 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:26:05,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/B/73dd4eba32d94662938a18ea92c810e4 is 50, key is test_row_0/B:col10/1732692362865/Put/seqid=0 2024-11-27T07:26:05,000 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cd6e5b712012f5d5b13dc2bcef88d1f:A 2024-11-27T07:26:05,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742193_1369 (size=12207) 2024-11-27T07:26:05,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:05,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51540 deadline: 1732692425029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:05,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:05,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51548 deadline: 1732692425029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:05,037 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 289954c9202d,42117,1732692259671 for 'test_row_2' in 9cd6e5b712012f5d5b13dc2bcef88d1f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T07:26:05,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:05,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:05,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:05,038 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 289954c9202d,42117,1732692259671 for 'test_row_0' in 9cd6e5b712012f5d5b13dc2bcef88d1f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T07:26:05,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:05,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:05,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:05,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742194_1370 (size=12001) 2024-11-27T07:26:05,041 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/B/73dd4eba32d94662938a18ea92c810e4 2024-11-27T07:26:05,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:05,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51584 deadline: 1732692425039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:05,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:05,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51586 deadline: 1732692425039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:05,049 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 289954c9202d,42117,1732692259671 for 'test_row_1' in 9cd6e5b712012f5d5b13dc2bcef88d1f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T07:26:05,050 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:05,050 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:05,049 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 289954c9202d,42117,1732692259671 for 'test_row_2' in 9cd6e5b712012f5d5b13dc2bcef88d1f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T07:26:05,050 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:05,050 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:05,050 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:05,050 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:05,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/C/a11c8c23870040b29c0d36e675d3bd6e is 50, key is test_row_0/C:col10/1732692362865/Put/seqid=0 2024-11-27T07:26:05,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742195_1371 (size=12001) 2024-11-27T07:26:05,063 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/C/a11c8c23870040b29c0d36e675d3bd6e 2024-11-27T07:26:05,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/6a6a3fd18132467ea5a621412a82cfba as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/6a6a3fd18132467ea5a621412a82cfba 2024-11-27T07:26:05,074 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/6a6a3fd18132467ea5a621412a82cfba, entries=150, sequenceid=116, filesize=30.2 K 2024-11-27T07:26:05,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/B/73dd4eba32d94662938a18ea92c810e4 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/73dd4eba32d94662938a18ea92c810e4 2024-11-27T07:26:05,079 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/73dd4eba32d94662938a18ea92c810e4, entries=150, sequenceid=116, filesize=11.7 K 2024-11-27T07:26:05,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/C/a11c8c23870040b29c0d36e675d3bd6e as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/a11c8c23870040b29c0d36e675d3bd6e 2024-11-27T07:26:05,083 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/a11c8c23870040b29c0d36e675d3bd6e, entries=150, sequenceid=116, filesize=11.7 K 2024-11-27T07:26:05,084 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 9cd6e5b712012f5d5b13dc2bcef88d1f in 588ms, sequenceid=116, compaction requested=false 2024-11-27T07:26:05,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 9cd6e5b712012f5d5b13dc2bcef88d1f: 2024-11-27T07:26:05,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:05,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-27T07:26:05,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-27T07:26:05,089 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-27T07:26:05,089 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9640 sec 2024-11-27T07:26:05,091 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.9700 sec 2024-11-27T07:26:05,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:26:05,241 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T07:26:05,241 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9cd6e5b712012f5d5b13dc2bcef88d1f 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-27T07:26:05,249 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127f4e94483e596483c8950f368812fc5b8_9cd6e5b712012f5d5b13dc2bcef88d1f is 50, key is test_row_0/A:col10/1732692364590/Put/seqid=0 2024-11-27T07:26:05,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-27T07:26:05,271 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T07:26:05,272 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T07:26:05,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-27T07:26:05,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-27T07:26:05,276 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T07:26:05,277 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T07:26:05,277 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T07:26:05,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742196_1372 (size=14794) 2024-11-27T07:26:05,285 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:26:05,288 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127f4e94483e596483c8950f368812fc5b8_9cd6e5b712012f5d5b13dc2bcef88d1f to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127f4e94483e596483c8950f368812fc5b8_9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:26:05,289 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/59984b5eadfb43a08b913fa21ad301aa, store: [table=TestAcidGuarantees family=A region=9cd6e5b712012f5d5b13dc2bcef88d1f] 2024-11-27T07:26:05,290 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/59984b5eadfb43a08b913fa21ad301aa is 175, key is test_row_0/A:col10/1732692364590/Put/seqid=0 2024-11-27T07:26:05,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742197_1373 (size=39749) 2024-11-27T07:26:05,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-27T07:26:05,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-27T07:26:05,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:05,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. as already flushing 2024-11-27T07:26:05,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:05,429 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:05,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:05,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:05,441 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/C/6b0dece1caba4c7bb71ae164750bce10 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/6b0dece1caba4c7bb71ae164750bce10 2024-11-27T07:26:05,452 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9cd6e5b712012f5d5b13dc2bcef88d1f/C of 9cd6e5b712012f5d5b13dc2bcef88d1f into 6b0dece1caba4c7bb71ae164750bce10(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:26:05,452 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cd6e5b712012f5d5b13dc2bcef88d1f: 2024-11-27T07:26:05,452 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., storeName=9cd6e5b712012f5d5b13dc2bcef88d1f/C, priority=13, startTime=1732692364495; duration=0sec 2024-11-27T07:26:05,452 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:26:05,452 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cd6e5b712012f5d5b13dc2bcef88d1f:C 2024-11-27T07:26:05,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:05,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51510 deadline: 1732692425452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:05,470 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:05,470 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:05,470 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:05,584 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-27T07:26:05,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:05,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. as already flushing 2024-11-27T07:26:05,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:05,585 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:05,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:05,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:05,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51510 deadline: 1732692425579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:05,588 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:05,588 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:05,588 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:05,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:05,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-27T07:26:05,741 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=135, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/59984b5eadfb43a08b913fa21ad301aa 2024-11-27T07:26:05,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-27T07:26:05,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:05,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. as already flushing 2024-11-27T07:26:05,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:05,741 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:05,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:05,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:05,763 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/B/de0511d5545344ecb3d968e50f363323 is 50, key is test_row_0/B:col10/1732692364590/Put/seqid=0 2024-11-27T07:26:05,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742198_1374 (size=12151) 2024-11-27T07:26:05,806 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/B/de0511d5545344ecb3d968e50f363323 2024-11-27T07:26:05,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:05,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51510 deadline: 1732692425799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:05,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:05,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:05,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:05,819 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/C/38be5e554bcb4481bcf642b9af3e8613 is 50, key is test_row_0/C:col10/1732692364590/Put/seqid=0 2024-11-27T07:26:05,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742199_1375 (size=12151) 2024-11-27T07:26:05,863 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/C/38be5e554bcb4481bcf642b9af3e8613 2024-11-27T07:26:05,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/59984b5eadfb43a08b913fa21ad301aa as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/59984b5eadfb43a08b913fa21ad301aa 2024-11-27T07:26:05,872 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/59984b5eadfb43a08b913fa21ad301aa, entries=200, sequenceid=135, filesize=38.8 K 2024-11-27T07:26:05,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/B/de0511d5545344ecb3d968e50f363323 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/de0511d5545344ecb3d968e50f363323 2024-11-27T07:26:05,885 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/de0511d5545344ecb3d968e50f363323, entries=150, sequenceid=135, filesize=11.9 K 2024-11-27T07:26:05,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/C/38be5e554bcb4481bcf642b9af3e8613 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/38be5e554bcb4481bcf642b9af3e8613 2024-11-27T07:26:05,898 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-27T07:26:05,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:05,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. as already flushing 2024-11-27T07:26:05,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:05,898 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:05,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:05,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:05,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-27T07:26:05,904 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/38be5e554bcb4481bcf642b9af3e8613, entries=150, sequenceid=135, filesize=11.9 K 2024-11-27T07:26:05,905 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 9cd6e5b712012f5d5b13dc2bcef88d1f in 664ms, sequenceid=135, compaction requested=true 2024-11-27T07:26:05,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9cd6e5b712012f5d5b13dc2bcef88d1f: 2024-11-27T07:26:05,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cd6e5b712012f5d5b13dc2bcef88d1f:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T07:26:05,906 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:26:05,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:26:05,906 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:26:05,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cd6e5b712012f5d5b13dc2bcef88d1f:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T07:26:05,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:26:05,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cd6e5b712012f5d5b13dc2bcef88d1f:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T07:26:05,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:26:05,907 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:26:05,908 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1541): 9cd6e5b712012f5d5b13dc2bcef88d1f/A is initiating minor compaction (all files) 2024-11-27T07:26:05,908 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cd6e5b712012f5d5b13dc2bcef88d1f/A in TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:05,908 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/abd923c0d5ac4a0e98edff57a47689f9, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/6a6a3fd18132467ea5a621412a82cfba, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/59984b5eadfb43a08b913fa21ad301aa] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp, totalSize=99.5 K 2024-11-27T07:26:05,908 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:05,908 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. files: [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/abd923c0d5ac4a0e98edff57a47689f9, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/6a6a3fd18132467ea5a621412a82cfba, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/59984b5eadfb43a08b913fa21ad301aa] 2024-11-27T07:26:05,908 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:26:05,908 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 9cd6e5b712012f5d5b13dc2bcef88d1f/B is initiating minor compaction (all files) 2024-11-27T07:26:05,908 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cd6e5b712012f5d5b13dc2bcef88d1f/B in TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:05,908 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/55689899011c4000a68c05ce335dbaa3, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/73dd4eba32d94662938a18ea92c810e4, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/de0511d5545344ecb3d968e50f363323] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp, totalSize=35.5 K 2024-11-27T07:26:05,909 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting abd923c0d5ac4a0e98edff57a47689f9, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732692362486 2024-11-27T07:26:05,909 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 55689899011c4000a68c05ce335dbaa3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732692362486 2024-11-27T07:26:05,909 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a6a3fd18132467ea5a621412a82cfba, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732692362836 2024-11-27T07:26:05,910 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 73dd4eba32d94662938a18ea92c810e4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732692362836 2024-11-27T07:26:05,910 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59984b5eadfb43a08b913fa21ad301aa, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732692364580 2024-11-27T07:26:05,910 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting de0511d5545344ecb3d968e50f363323, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732692364580 2024-11-27T07:26:05,924 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9cd6e5b712012f5d5b13dc2bcef88d1f] 2024-11-27T07:26:05,926 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cd6e5b712012f5d5b13dc2bcef88d1f#B#compaction#316 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:26:05,927 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/B/f60b55186b49468eb95e865163518312 is 50, key is test_row_0/B:col10/1732692364590/Put/seqid=0 2024-11-27T07:26:05,937 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112766bc4e408f66444ea89409044d5ac2de_9cd6e5b712012f5d5b13dc2bcef88d1f store=[table=TestAcidGuarantees family=A region=9cd6e5b712012f5d5b13dc2bcef88d1f] 2024-11-27T07:26:05,939 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112766bc4e408f66444ea89409044d5ac2de_9cd6e5b712012f5d5b13dc2bcef88d1f, store=[table=TestAcidGuarantees family=A region=9cd6e5b712012f5d5b13dc2bcef88d1f] 2024-11-27T07:26:05,939 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112766bc4e408f66444ea89409044d5ac2de_9cd6e5b712012f5d5b13dc2bcef88d1f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9cd6e5b712012f5d5b13dc2bcef88d1f] 2024-11-27T07:26:05,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742200_1376 (size=12459) 2024-11-27T07:26:05,953 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/B/f60b55186b49468eb95e865163518312 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/f60b55186b49468eb95e865163518312 2024-11-27T07:26:05,964 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9cd6e5b712012f5d5b13dc2bcef88d1f/B of 9cd6e5b712012f5d5b13dc2bcef88d1f into f60b55186b49468eb95e865163518312(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:26:05,964 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cd6e5b712012f5d5b13dc2bcef88d1f: 2024-11-27T07:26:05,964 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., storeName=9cd6e5b712012f5d5b13dc2bcef88d1f/B, priority=13, startTime=1732692365906; duration=0sec 2024-11-27T07:26:05,964 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:26:05,964 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cd6e5b712012f5d5b13dc2bcef88d1f:B 2024-11-27T07:26:05,964 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:26:05,965 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:26:05,965 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 9cd6e5b712012f5d5b13dc2bcef88d1f/C is initiating minor compaction (all files) 2024-11-27T07:26:05,965 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cd6e5b712012f5d5b13dc2bcef88d1f/C in TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:05,966 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/6b0dece1caba4c7bb71ae164750bce10, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/a11c8c23870040b29c0d36e675d3bd6e, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/38be5e554bcb4481bcf642b9af3e8613] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp, totalSize=35.5 K 2024-11-27T07:26:05,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742201_1377 (size=4469) 2024-11-27T07:26:05,967 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b0dece1caba4c7bb71ae164750bce10, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732692362486 2024-11-27T07:26:05,968 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting a11c8c23870040b29c0d36e675d3bd6e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732692362836 2024-11-27T07:26:05,972 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 38be5e554bcb4481bcf642b9af3e8613, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732692364580 2024-11-27T07:26:05,980 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cd6e5b712012f5d5b13dc2bcef88d1f#C#compaction#317 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:26:05,981 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/C/180862389c0c4704a09325d111a11770 is 50, key is test_row_0/C:col10/1732692364590/Put/seqid=0 2024-11-27T07:26:06,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742202_1378 (size=12459) 2024-11-27T07:26:06,052 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-27T07:26:06,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:06,053 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 9cd6e5b712012f5d5b13dc2bcef88d1f 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-27T07:26:06,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112767ede66e254e4044ba13cb2a9a220291_9cd6e5b712012f5d5b13dc2bcef88d1f is 50, key is test_row_0/A:col10/1732692365409/Put/seqid=0 2024-11-27T07:26:06,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742203_1379 (size=12304) 2024-11-27T07:26:06,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:26:06,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. as already flushing 2024-11-27T07:26:06,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:06,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51510 deadline: 1732692426226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:06,238 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:06,238 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:06,238 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:06,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:06,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51510 deadline: 1732692426350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:06,362 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:06,362 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:06,362 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:06,369 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cd6e5b712012f5d5b13dc2bcef88d1f#A#compaction#315 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:26:06,369 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/9a6f3fe690ef480187450b400a823e1e is 175, key is test_row_0/A:col10/1732692364590/Put/seqid=0 2024-11-27T07:26:06,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742204_1380 (size=31413) 2024-11-27T07:26:06,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-27T07:26:06,420 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/9a6f3fe690ef480187450b400a823e1e as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/9a6f3fe690ef480187450b400a823e1e 2024-11-27T07:26:06,427 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/C/180862389c0c4704a09325d111a11770 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/180862389c0c4704a09325d111a11770 2024-11-27T07:26:06,436 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9cd6e5b712012f5d5b13dc2bcef88d1f/A of 9cd6e5b712012f5d5b13dc2bcef88d1f into 9a6f3fe690ef480187450b400a823e1e(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:26:06,436 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cd6e5b712012f5d5b13dc2bcef88d1f: 2024-11-27T07:26:06,436 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., storeName=9cd6e5b712012f5d5b13dc2bcef88d1f/A, priority=13, startTime=1732692365905; duration=0sec 2024-11-27T07:26:06,438 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:26:06,438 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cd6e5b712012f5d5b13dc2bcef88d1f:A 2024-11-27T07:26:06,441 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9cd6e5b712012f5d5b13dc2bcef88d1f/C of 9cd6e5b712012f5d5b13dc2bcef88d1f into 180862389c0c4704a09325d111a11770(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:26:06,441 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cd6e5b712012f5d5b13dc2bcef88d1f: 2024-11-27T07:26:06,441 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., storeName=9cd6e5b712012f5d5b13dc2bcef88d1f/C, priority=13, startTime=1732692365906; duration=0sec 2024-11-27T07:26:06,441 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:26:06,441 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cd6e5b712012f5d5b13dc2bcef88d1f:C 2024-11-27T07:26:06,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:26:06,490 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112767ede66e254e4044ba13cb2a9a220291_9cd6e5b712012f5d5b13dc2bcef88d1f to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112767ede66e254e4044ba13cb2a9a220291_9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:26:06,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/f182720f868b40d1863db13ed6257f5c, store: [table=TestAcidGuarantees family=A region=9cd6e5b712012f5d5b13dc2bcef88d1f] 2024-11-27T07:26:06,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/f182720f868b40d1863db13ed6257f5c is 175, key is test_row_0/A:col10/1732692365409/Put/seqid=0 2024-11-27T07:26:06,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742205_1381 (size=31105) 2024-11-27T07:26:06,530 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=156, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/f182720f868b40d1863db13ed6257f5c 2024-11-27T07:26:06,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/B/83d1fa12abe247d4b9862bab9d3e695c is 50, key is test_row_0/B:col10/1732692365409/Put/seqid=0 2024-11-27T07:26:06,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:06,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51510 deadline: 1732692426569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:06,574 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:06,574 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:06,574 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:06,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742206_1382 (size=12151) 2024-11-27T07:26:06,580 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/B/83d1fa12abe247d4b9862bab9d3e695c 2024-11-27T07:26:06,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/C/357dc42b39c048c6a42c3c684d7499db is 50, key is test_row_0/C:col10/1732692365409/Put/seqid=0 2024-11-27T07:26:06,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742207_1383 (size=12151) 2024-11-27T07:26:06,636 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/C/357dc42b39c048c6a42c3c684d7499db 2024-11-27T07:26:06,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/f182720f868b40d1863db13ed6257f5c as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/f182720f868b40d1863db13ed6257f5c 2024-11-27T07:26:06,655 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/f182720f868b40d1863db13ed6257f5c, entries=150, sequenceid=156, filesize=30.4 K 2024-11-27T07:26:06,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/B/83d1fa12abe247d4b9862bab9d3e695c as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/83d1fa12abe247d4b9862bab9d3e695c 2024-11-27T07:26:06,663 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/83d1fa12abe247d4b9862bab9d3e695c, entries=150, sequenceid=156, filesize=11.9 K 2024-11-27T07:26:06,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/C/357dc42b39c048c6a42c3c684d7499db as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/357dc42b39c048c6a42c3c684d7499db 2024-11-27T07:26:06,670 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/357dc42b39c048c6a42c3c684d7499db, entries=150, sequenceid=156, filesize=11.9 K 2024-11-27T07:26:06,675 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 9cd6e5b712012f5d5b13dc2bcef88d1f in 622ms, sequenceid=156, compaction requested=false 2024-11-27T07:26:06,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 9cd6e5b712012f5d5b13dc2bcef88d1f: 2024-11-27T07:26:06,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:06,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-27T07:26:06,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-27T07:26:06,679 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-27T07:26:06,679 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4000 sec 2024-11-27T07:26:06,682 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 1.4090 sec 2024-11-27T07:26:06,889 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T07:26:06,889 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9cd6e5b712012f5d5b13dc2bcef88d1f 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-27T07:26:06,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:26:06,911 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411273e62b0c25fd04adeaf09b03d42c77d03_9cd6e5b712012f5d5b13dc2bcef88d1f is 50, key is test_row_0/A:col10/1732692366884/Put/seqid=0 2024-11-27T07:26:06,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742208_1384 (size=14794) 2024-11-27T07:26:07,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:07,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51510 deadline: 1732692427049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:07,068 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:07,068 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:07,068 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:07,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:07,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51586 deadline: 1732692427059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:07,074 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 289954c9202d,42117,1732692259671 for 'test_row_2' in 9cd6e5b712012f5d5b13dc2bcef88d1f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T07:26:07,074 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:07,074 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:07,074 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:07,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:07,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51540 deadline: 1732692427066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:07,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:07,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:07,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:07,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:07,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51548 deadline: 1732692427067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:07,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:07,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:07,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:07,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:07,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51584 deadline: 1732692427070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:07,088 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 289954c9202d,42117,1732692259671 for 'test_row_1' in 9cd6e5b712012f5d5b13dc2bcef88d1f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T07:26:07,088 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:07,088 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:07,088 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:07,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:07,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51510 deadline: 1732692427179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:07,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:07,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:07,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:07,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:07,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51540 deadline: 1732692427179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:07,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:07,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:07,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:07,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:07,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51548 deadline: 1732692427190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:07,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:07,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:07,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:07,354 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:26:07,358 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411273e62b0c25fd04adeaf09b03d42c77d03_9cd6e5b712012f5d5b13dc2bcef88d1f to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411273e62b0c25fd04adeaf09b03d42c77d03_9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:26:07,358 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/b3f0410d1ddb488ca6cb753ca8f4643a, store: [table=TestAcidGuarantees family=A region=9cd6e5b712012f5d5b13dc2bcef88d1f] 2024-11-27T07:26:07,359 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/b3f0410d1ddb488ca6cb753ca8f4643a is 175, key is test_row_0/A:col10/1732692366884/Put/seqid=0 2024-11-27T07:26:07,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742209_1385 (size=39749) 2024-11-27T07:26:07,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:07,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51510 deadline: 1732692427399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:07,407 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:07,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:07,408 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:07,408 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:07,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51540 deadline: 1732692427399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:07,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:07,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:07,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:07,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:07,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51548 deadline: 1732692427409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:07,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:07,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:07,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:07,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-27T07:26:07,420 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T07:26:07,420 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T07:26:07,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-27T07:26:07,423 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T07:26:07,423 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T07:26:07,424 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T07:26:07,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-27T07:26:07,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-27T07:26:07,577 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-27T07:26:07,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:07,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. as already flushing 2024-11-27T07:26:07,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:07,577 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:07,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:07,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:07,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:07,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51510 deadline: 1732692427709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:07,714 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:07,714 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:07,714 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:07,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:07,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51540 deadline: 1732692427720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:07,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:07,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:07,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:07,730 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-27T07:26:07,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:07,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. as already flushing 2024-11-27T07:26:07,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:07,730 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:07,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:07,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:07,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-27T07:26:07,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:07,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51548 deadline: 1732692427729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:07,741 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:07,741 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:07,741 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:07,767 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=176, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/b3f0410d1ddb488ca6cb753ca8f4643a 2024-11-27T07:26:07,779 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/B/c44723700fc443e9b60b32204c16e7ab is 50, key is test_row_0/B:col10/1732692366884/Put/seqid=0 2024-11-27T07:26:07,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742210_1386 (size=12151) 2024-11-27T07:26:07,793 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/B/c44723700fc443e9b60b32204c16e7ab 2024-11-27T07:26:07,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/C/681ccb68fa3c4f4fb56b8204d056b6a9 is 50, key is test_row_0/C:col10/1732692366884/Put/seqid=0 2024-11-27T07:26:07,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742211_1387 (size=12151) 2024-11-27T07:26:07,819 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/C/681ccb68fa3c4f4fb56b8204d056b6a9 2024-11-27T07:26:07,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/b3f0410d1ddb488ca6cb753ca8f4643a as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/b3f0410d1ddb488ca6cb753ca8f4643a 2024-11-27T07:26:07,826 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/b3f0410d1ddb488ca6cb753ca8f4643a, entries=200, sequenceid=176, filesize=38.8 K 2024-11-27T07:26:07,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/B/c44723700fc443e9b60b32204c16e7ab as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/c44723700fc443e9b60b32204c16e7ab 2024-11-27T07:26:07,830 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/c44723700fc443e9b60b32204c16e7ab, entries=150, sequenceid=176, filesize=11.9 K 2024-11-27T07:26:07,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/C/681ccb68fa3c4f4fb56b8204d056b6a9 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/681ccb68fa3c4f4fb56b8204d056b6a9 2024-11-27T07:26:07,834 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/681ccb68fa3c4f4fb56b8204d056b6a9, entries=150, sequenceid=176, filesize=11.9 K 2024-11-27T07:26:07,835 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 9cd6e5b712012f5d5b13dc2bcef88d1f in 946ms, sequenceid=176, compaction requested=true 2024-11-27T07:26:07,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9cd6e5b712012f5d5b13dc2bcef88d1f: 2024-11-27T07:26:07,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cd6e5b712012f5d5b13dc2bcef88d1f:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T07:26:07,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:26:07,835 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:26:07,835 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:26:07,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cd6e5b712012f5d5b13dc2bcef88d1f:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T07:26:07,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:26:07,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cd6e5b712012f5d5b13dc2bcef88d1f:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T07:26:07,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:26:07,836 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:26:07,836 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102267 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:26:07,836 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1541): 9cd6e5b712012f5d5b13dc2bcef88d1f/A is initiating minor compaction (all files) 2024-11-27T07:26:07,836 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 9cd6e5b712012f5d5b13dc2bcef88d1f/B is initiating minor compaction (all files) 2024-11-27T07:26:07,836 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cd6e5b712012f5d5b13dc2bcef88d1f/B in TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:07,836 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cd6e5b712012f5d5b13dc2bcef88d1f/A in TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:07,836 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/f60b55186b49468eb95e865163518312, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/83d1fa12abe247d4b9862bab9d3e695c, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/c44723700fc443e9b60b32204c16e7ab] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp, totalSize=35.9 K 2024-11-27T07:26:07,836 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/9a6f3fe690ef480187450b400a823e1e, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/f182720f868b40d1863db13ed6257f5c, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/b3f0410d1ddb488ca6cb753ca8f4643a] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp, totalSize=99.9 K 2024-11-27T07:26:07,836 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:07,836 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. files: [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/9a6f3fe690ef480187450b400a823e1e, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/f182720f868b40d1863db13ed6257f5c, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/b3f0410d1ddb488ca6cb753ca8f4643a] 2024-11-27T07:26:07,837 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a6f3fe690ef480187450b400a823e1e, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732692364580 2024-11-27T07:26:07,837 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting f60b55186b49468eb95e865163518312, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732692364580 2024-11-27T07:26:07,838 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting f182720f868b40d1863db13ed6257f5c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732692365409 2024-11-27T07:26:07,838 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 83d1fa12abe247d4b9862bab9d3e695c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732692365409 2024-11-27T07:26:07,838 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3f0410d1ddb488ca6cb753ca8f4643a, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732692366207 2024-11-27T07:26:07,838 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting c44723700fc443e9b60b32204c16e7ab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732692366207 2024-11-27T07:26:07,849 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9cd6e5b712012f5d5b13dc2bcef88d1f] 2024-11-27T07:26:07,850 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cd6e5b712012f5d5b13dc2bcef88d1f#B#compaction#324 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:26:07,851 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/B/e1f4ab66aa934ef79b97754385f15b18 is 50, key is test_row_0/B:col10/1732692366884/Put/seqid=0 2024-11-27T07:26:07,857 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112775953b7c21ff4976ae229aac5258f610_9cd6e5b712012f5d5b13dc2bcef88d1f store=[table=TestAcidGuarantees family=A region=9cd6e5b712012f5d5b13dc2bcef88d1f] 2024-11-27T07:26:07,858 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112775953b7c21ff4976ae229aac5258f610_9cd6e5b712012f5d5b13dc2bcef88d1f, store=[table=TestAcidGuarantees family=A region=9cd6e5b712012f5d5b13dc2bcef88d1f] 2024-11-27T07:26:07,858 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112775953b7c21ff4976ae229aac5258f610_9cd6e5b712012f5d5b13dc2bcef88d1f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9cd6e5b712012f5d5b13dc2bcef88d1f] 2024-11-27T07:26:07,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742213_1389 (size=4469) 2024-11-27T07:26:07,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742212_1388 (size=12561) 2024-11-27T07:26:07,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-27T07:26:07,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:07,885 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 9cd6e5b712012f5d5b13dc2bcef88d1f 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-27T07:26:07,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127e1b60cee38634529bf6c4f6a8da5a47a_9cd6e5b712012f5d5b13dc2bcef88d1f is 50, key is test_row_0/A:col10/1732692367065/Put/seqid=0 2024-11-27T07:26:07,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742214_1390 (size=12304) 2024-11-27T07:26:08,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-27T07:26:08,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:26:08,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. as already flushing 2024-11-27T07:26:08,275 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cd6e5b712012f5d5b13dc2bcef88d1f#A#compaction#325 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:26:08,275 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/7c148dff92d148eba5876c2f76497948 is 175, key is test_row_0/A:col10/1732692366884/Put/seqid=0 2024-11-27T07:26:08,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742215_1391 (size=31515) 2024-11-27T07:26:08,283 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/B/e1f4ab66aa934ef79b97754385f15b18 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/e1f4ab66aa934ef79b97754385f15b18 2024-11-27T07:26:08,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:08,287 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9cd6e5b712012f5d5b13dc2bcef88d1f/B of 9cd6e5b712012f5d5b13dc2bcef88d1f into e1f4ab66aa934ef79b97754385f15b18(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:26:08,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51540 deadline: 1732692428279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:08,287 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cd6e5b712012f5d5b13dc2bcef88d1f: 2024-11-27T07:26:08,287 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., storeName=9cd6e5b712012f5d5b13dc2bcef88d1f/B, priority=13, startTime=1732692367835; duration=0sec 2024-11-27T07:26:08,287 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:26:08,287 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cd6e5b712012f5d5b13dc2bcef88d1f:B 2024-11-27T07:26:08,287 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:26:08,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:08,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:08,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:08,287 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/7c148dff92d148eba5876c2f76497948 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/7c148dff92d148eba5876c2f76497948 2024-11-27T07:26:08,288 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:26:08,288 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 9cd6e5b712012f5d5b13dc2bcef88d1f/C is initiating minor compaction (all files) 2024-11-27T07:26:08,288 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cd6e5b712012f5d5b13dc2bcef88d1f/C in TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:08,288 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/180862389c0c4704a09325d111a11770, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/357dc42b39c048c6a42c3c684d7499db, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/681ccb68fa3c4f4fb56b8204d056b6a9] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp, totalSize=35.9 K 2024-11-27T07:26:08,288 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 180862389c0c4704a09325d111a11770, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732692364580 2024-11-27T07:26:08,289 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 357dc42b39c048c6a42c3c684d7499db, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732692365409 2024-11-27T07:26:08,289 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 681ccb68fa3c4f4fb56b8204d056b6a9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732692366207 2024-11-27T07:26:08,291 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9cd6e5b712012f5d5b13dc2bcef88d1f/A of 9cd6e5b712012f5d5b13dc2bcef88d1f into 7c148dff92d148eba5876c2f76497948(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:26:08,291 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cd6e5b712012f5d5b13dc2bcef88d1f: 2024-11-27T07:26:08,291 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., storeName=9cd6e5b712012f5d5b13dc2bcef88d1f/A, priority=13, startTime=1732692367835; duration=0sec 2024-11-27T07:26:08,292 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:26:08,292 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cd6e5b712012f5d5b13dc2bcef88d1f:A 2024-11-27T07:26:08,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:08,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51510 deadline: 1732692428282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:08,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:08,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:08,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:08,297 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cd6e5b712012f5d5b13dc2bcef88d1f#C#compaction#327 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:26:08,297 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/C/8f4d1b717d9d46f98d1bde8ec649ffe9 is 50, key is test_row_0/C:col10/1732692366884/Put/seqid=0 2024-11-27T07:26:08,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:08,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51548 deadline: 1732692428287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:08,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:08,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:08,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:08,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742216_1392 (size=12561) 2024-11-27T07:26:08,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:26:08,318 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127e1b60cee38634529bf6c4f6a8da5a47a_9cd6e5b712012f5d5b13dc2bcef88d1f to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127e1b60cee38634529bf6c4f6a8da5a47a_9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:26:08,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/cd39876837ca4544ad3f01a3d9d9d856, store: [table=TestAcidGuarantees family=A region=9cd6e5b712012f5d5b13dc2bcef88d1f] 2024-11-27T07:26:08,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/cd39876837ca4544ad3f01a3d9d9d856 is 175, key is test_row_0/A:col10/1732692367065/Put/seqid=0 2024-11-27T07:26:08,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742217_1393 (size=31105) 2024-11-27T07:26:08,331 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=194, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/cd39876837ca4544ad3f01a3d9d9d856 2024-11-27T07:26:08,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/B/89bfc11864da4ed391e93b8e72c0683e is 50, key is test_row_0/B:col10/1732692367065/Put/seqid=0 2024-11-27T07:26:08,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742218_1394 (size=12151) 2024-11-27T07:26:08,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:08,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51540 deadline: 1732692428389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:08,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:08,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:08,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:08,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:08,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51510 deadline: 1732692428399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:08,403 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:08,403 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:08,403 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:08,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:08,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51548 deadline: 1732692428409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:08,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:08,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:08,415 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:08,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-27T07:26:08,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:08,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51540 deadline: 1732692428610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:08,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:08,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51510 deadline: 1732692428610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:08,613 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:08,613 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:08,613 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:08,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:08,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:08,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:08,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:08,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51548 deadline: 1732692428621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:08,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:08,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:08,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:08,712 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/C/8f4d1b717d9d46f98d1bde8ec649ffe9 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/8f4d1b717d9d46f98d1bde8ec649ffe9 2024-11-27T07:26:08,718 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9cd6e5b712012f5d5b13dc2bcef88d1f/C of 9cd6e5b712012f5d5b13dc2bcef88d1f into 8f4d1b717d9d46f98d1bde8ec649ffe9(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:26:08,718 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cd6e5b712012f5d5b13dc2bcef88d1f: 2024-11-27T07:26:08,718 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., storeName=9cd6e5b712012f5d5b13dc2bcef88d1f/C, priority=13, startTime=1732692367835; duration=0sec 2024-11-27T07:26:08,718 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:26:08,718 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cd6e5b712012f5d5b13dc2bcef88d1f:C 2024-11-27T07:26:08,752 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/B/89bfc11864da4ed391e93b8e72c0683e 2024-11-27T07:26:08,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/C/2f8b97ce05434d189b0879d1c38c9318 is 50, key is test_row_0/C:col10/1732692367065/Put/seqid=0 2024-11-27T07:26:08,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742219_1395 (size=12151) 2024-11-27T07:26:08,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:08,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51510 deadline: 1732692428919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:08,920 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:08,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:08,920 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:08,920 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:08,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51540 deadline: 1732692428919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:08,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:08,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:08,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:08,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:08,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51548 deadline: 1732692428939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:08,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:08,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:08,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:09,186 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/C/2f8b97ce05434d189b0879d1c38c9318 2024-11-27T07:26:09,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/cd39876837ca4544ad3f01a3d9d9d856 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/cd39876837ca4544ad3f01a3d9d9d856 2024-11-27T07:26:09,194 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/cd39876837ca4544ad3f01a3d9d9d856, entries=150, sequenceid=194, filesize=30.4 K 2024-11-27T07:26:09,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/B/89bfc11864da4ed391e93b8e72c0683e as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/89bfc11864da4ed391e93b8e72c0683e 2024-11-27T07:26:09,199 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/89bfc11864da4ed391e93b8e72c0683e, entries=150, sequenceid=194, filesize=11.9 K 2024-11-27T07:26:09,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/C/2f8b97ce05434d189b0879d1c38c9318 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/2f8b97ce05434d189b0879d1c38c9318 2024-11-27T07:26:09,203 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/2f8b97ce05434d189b0879d1c38c9318, entries=150, sequenceid=194, filesize=11.9 K 2024-11-27T07:26:09,204 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 9cd6e5b712012f5d5b13dc2bcef88d1f in 1320ms, sequenceid=194, compaction requested=false 2024-11-27T07:26:09,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 9cd6e5b712012f5d5b13dc2bcef88d1f: 2024-11-27T07:26:09,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:09,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-27T07:26:09,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-27T07:26:09,207 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-27T07:26:09,207 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7830 sec 2024-11-27T07:26:09,209 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.7880 sec 2024-11-27T07:26:09,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:26:09,434 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T07:26:09,435 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9cd6e5b712012f5d5b13dc2bcef88d1f 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-27T07:26:09,448 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112708eac5f3d9f84c87a102100fb292c58f_9cd6e5b712012f5d5b13dc2bcef88d1f is 50, key is test_row_0/A:col10/1732692369433/Put/seqid=0 2024-11-27T07:26:09,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742220_1396 (size=14794) 2024-11-27T07:26:09,472 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:26:09,477 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112708eac5f3d9f84c87a102100fb292c58f_9cd6e5b712012f5d5b13dc2bcef88d1f to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112708eac5f3d9f84c87a102100fb292c58f_9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:26:09,478 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/d459f223b77742688e098b03db0ef9b7, store: [table=TestAcidGuarantees family=A region=9cd6e5b712012f5d5b13dc2bcef88d1f] 2024-11-27T07:26:09,478 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/d459f223b77742688e098b03db0ef9b7 is 175, key is test_row_0/A:col10/1732692369433/Put/seqid=0 2024-11-27T07:26:09,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:09,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51510 deadline: 1732692429484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:09,487 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:09,487 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:09,487 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:09,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742221_1397 (size=39749) 2024-11-27T07:26:09,490 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=217, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/d459f223b77742688e098b03db0ef9b7 2024-11-27T07:26:09,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:09,496 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/B/5f3b9207350a49a3aa515aacd6429753 is 50, key is test_row_0/B:col10/1732692369433/Put/seqid=0 2024-11-27T07:26:09,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51548 deadline: 1732692429485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:09,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:09,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:09,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:09,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:09,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51540 deadline: 1732692429486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:09,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:09,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:09,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:09,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742222_1398 (size=12151) 2024-11-27T07:26:09,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-27T07:26:09,569 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T07:26:09,570 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T07:26:09,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-27T07:26:09,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-27T07:26:09,577 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T07:26:09,578 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T07:26:09,578 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T07:26:09,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:09,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51510 deadline: 1732692429589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:09,600 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:09,600 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:09,600 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:09,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:09,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51548 deadline: 1732692429599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:09,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:09,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:09,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:09,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:09,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51540 deadline: 1732692429599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:09,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:09,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:09,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:09,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-27T07:26:09,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-27T07:26:09,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:09,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. as already flushing 2024-11-27T07:26:09,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:09,731 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:09,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:09,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:09,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:09,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51510 deadline: 1732692429809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:09,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:09,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:09,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:09,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:09,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51548 deadline: 1732692429819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:09,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:09,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:09,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:09,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:09,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51540 deadline: 1732692429819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:09,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:09,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:09,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:09,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-27T07:26:09,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:09,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. as already flushing 2024-11-27T07:26:09,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:09,884 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:09,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:09,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:09,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-27T07:26:09,906 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/B/5f3b9207350a49a3aa515aacd6429753 2024-11-27T07:26:09,913 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/C/8e6aa46a99164a88abe3527abb210acb is 50, key is test_row_0/C:col10/1732692369433/Put/seqid=0 2024-11-27T07:26:09,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742223_1399 (size=12151) 2024-11-27T07:26:09,921 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/C/8e6aa46a99164a88abe3527abb210acb 2024-11-27T07:26:09,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/d459f223b77742688e098b03db0ef9b7 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/d459f223b77742688e098b03db0ef9b7 2024-11-27T07:26:09,935 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/d459f223b77742688e098b03db0ef9b7, entries=200, sequenceid=217, filesize=38.8 K 2024-11-27T07:26:09,936 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/B/5f3b9207350a49a3aa515aacd6429753 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/5f3b9207350a49a3aa515aacd6429753 2024-11-27T07:26:09,941 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/5f3b9207350a49a3aa515aacd6429753, entries=150, sequenceid=217, filesize=11.9 K 2024-11-27T07:26:09,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/C/8e6aa46a99164a88abe3527abb210acb as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/8e6aa46a99164a88abe3527abb210acb 2024-11-27T07:26:09,953 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/8e6aa46a99164a88abe3527abb210acb, entries=150, sequenceid=217, filesize=11.9 K 2024-11-27T07:26:09,957 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 9cd6e5b712012f5d5b13dc2bcef88d1f in 523ms, sequenceid=217, compaction requested=true 2024-11-27T07:26:09,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9cd6e5b712012f5d5b13dc2bcef88d1f: 2024-11-27T07:26:09,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cd6e5b712012f5d5b13dc2bcef88d1f:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T07:26:09,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:26:09,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cd6e5b712012f5d5b13dc2bcef88d1f:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T07:26:09,958 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:26:09,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:26:09,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cd6e5b712012f5d5b13dc2bcef88d1f:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T07:26:09,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:26:09,958 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:26:09,959 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102369 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:26:09,959 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:26:09,959 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 9cd6e5b712012f5d5b13dc2bcef88d1f/A is initiating minor compaction (all files) 2024-11-27T07:26:09,959 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1541): 9cd6e5b712012f5d5b13dc2bcef88d1f/B is initiating minor compaction (all files) 2024-11-27T07:26:09,959 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cd6e5b712012f5d5b13dc2bcef88d1f/B in TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:09,959 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cd6e5b712012f5d5b13dc2bcef88d1f/A in TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:09,959 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/e1f4ab66aa934ef79b97754385f15b18, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/89bfc11864da4ed391e93b8e72c0683e, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/5f3b9207350a49a3aa515aacd6429753] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp, totalSize=36.0 K 2024-11-27T07:26:09,959 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/7c148dff92d148eba5876c2f76497948, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/cd39876837ca4544ad3f01a3d9d9d856, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/d459f223b77742688e098b03db0ef9b7] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp, totalSize=100.0 K 2024-11-27T07:26:09,959 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:09,959 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. files: [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/7c148dff92d148eba5876c2f76497948, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/cd39876837ca4544ad3f01a3d9d9d856, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/d459f223b77742688e098b03db0ef9b7] 2024-11-27T07:26:09,959 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c148dff92d148eba5876c2f76497948, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732692366207 2024-11-27T07:26:09,959 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1f4ab66aa934ef79b97754385f15b18, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732692366207 2024-11-27T07:26:09,960 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89bfc11864da4ed391e93b8e72c0683e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732692367045 2024-11-27T07:26:09,960 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting cd39876837ca4544ad3f01a3d9d9d856, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732692367045 2024-11-27T07:26:09,960 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f3b9207350a49a3aa515aacd6429753, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1732692368285 2024-11-27T07:26:09,960 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting d459f223b77742688e098b03db0ef9b7, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1732692368270 2024-11-27T07:26:09,978 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cd6e5b712012f5d5b13dc2bcef88d1f#B#compaction#333 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:26:09,979 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/B/222bb4458c734a6fba1fb95730dec978 is 50, key is test_row_0/B:col10/1732692369433/Put/seqid=0 2024-11-27T07:26:09,989 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9cd6e5b712012f5d5b13dc2bcef88d1f] 2024-11-27T07:26:10,009 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411271889772326114976a4d5314af6133120_9cd6e5b712012f5d5b13dc2bcef88d1f store=[table=TestAcidGuarantees family=A region=9cd6e5b712012f5d5b13dc2bcef88d1f] 2024-11-27T07:26:10,011 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411271889772326114976a4d5314af6133120_9cd6e5b712012f5d5b13dc2bcef88d1f, store=[table=TestAcidGuarantees family=A region=9cd6e5b712012f5d5b13dc2bcef88d1f] 2024-11-27T07:26:10,011 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411271889772326114976a4d5314af6133120_9cd6e5b712012f5d5b13dc2bcef88d1f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9cd6e5b712012f5d5b13dc2bcef88d1f] 2024-11-27T07:26:10,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742224_1400 (size=12663) 2024-11-27T07:26:10,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742225_1401 (size=4469) 2024-11-27T07:26:10,038 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-27T07:26:10,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:10,039 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 9cd6e5b712012f5d5b13dc2bcef88d1f 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-27T07:26:10,042 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cd6e5b712012f5d5b13dc2bcef88d1f#A#compaction#334 average throughput is 0.46 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:26:10,043 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/6b1f7f00b7c84502bc314d8211a2d2e7 is 175, key is test_row_0/A:col10/1732692369433/Put/seqid=0 2024-11-27T07:26:10,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112766367639368a4b9c9e887d7b74bafa8b_9cd6e5b712012f5d5b13dc2bcef88d1f is 50, key is test_row_0/A:col10/1732692369484/Put/seqid=0 2024-11-27T07:26:10,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742226_1402 (size=31617) 2024-11-27T07:26:10,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742227_1403 (size=12304) 2024-11-27T07:26:10,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:26:10,133 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. as already flushing 2024-11-27T07:26:10,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-27T07:26:10,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:10,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51540 deadline: 1732692430197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:10,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:10,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:10,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:10,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:10,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51548 deadline: 1732692430208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:10,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:10,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51510 deadline: 1732692430208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:10,223 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:10,223 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:10,223 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:10,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:10,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:10,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:10,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:10,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51540 deadline: 1732692430319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:10,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:10,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:10,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:10,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:10,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51510 deadline: 1732692430329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:10,337 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:10,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:10,337 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:10,337 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:10,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51548 deadline: 1732692430330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:10,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:10,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:10,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:10,422 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/B/222bb4458c734a6fba1fb95730dec978 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/222bb4458c734a6fba1fb95730dec978 2024-11-27T07:26:10,426 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9cd6e5b712012f5d5b13dc2bcef88d1f/B of 9cd6e5b712012f5d5b13dc2bcef88d1f into 222bb4458c734a6fba1fb95730dec978(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:26:10,426 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cd6e5b712012f5d5b13dc2bcef88d1f: 2024-11-27T07:26:10,426 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., storeName=9cd6e5b712012f5d5b13dc2bcef88d1f/B, priority=13, startTime=1732692369958; duration=0sec 2024-11-27T07:26:10,426 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:26:10,426 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cd6e5b712012f5d5b13dc2bcef88d1f:B 2024-11-27T07:26:10,426 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:26:10,427 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:26:10,427 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1541): 9cd6e5b712012f5d5b13dc2bcef88d1f/C is initiating minor compaction (all files) 2024-11-27T07:26:10,428 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cd6e5b712012f5d5b13dc2bcef88d1f/C in TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:10,428 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/8f4d1b717d9d46f98d1bde8ec649ffe9, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/2f8b97ce05434d189b0879d1c38c9318, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/8e6aa46a99164a88abe3527abb210acb] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp, totalSize=36.0 K 2024-11-27T07:26:10,428 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f4d1b717d9d46f98d1bde8ec649ffe9, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732692366207 2024-11-27T07:26:10,428 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f8b97ce05434d189b0879d1c38c9318, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732692367045 2024-11-27T07:26:10,429 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e6aa46a99164a88abe3527abb210acb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1732692368285 2024-11-27T07:26:10,435 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cd6e5b712012f5d5b13dc2bcef88d1f#C#compaction#336 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:26:10,435 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/C/58c3117356904f06b3bc4b9a216f4a6e is 50, key is test_row_0/C:col10/1732692369433/Put/seqid=0 2024-11-27T07:26:10,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742228_1404 (size=12663) 2024-11-27T07:26:10,486 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/6b1f7f00b7c84502bc314d8211a2d2e7 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/6b1f7f00b7c84502bc314d8211a2d2e7 2024-11-27T07:26:10,490 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9cd6e5b712012f5d5b13dc2bcef88d1f/A of 9cd6e5b712012f5d5b13dc2bcef88d1f into 6b1f7f00b7c84502bc314d8211a2d2e7(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:26:10,490 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cd6e5b712012f5d5b13dc2bcef88d1f: 2024-11-27T07:26:10,490 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., storeName=9cd6e5b712012f5d5b13dc2bcef88d1f/A, priority=13, startTime=1732692369957; duration=0sec 2024-11-27T07:26:10,490 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:26:10,490 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cd6e5b712012f5d5b13dc2bcef88d1f:A 2024-11-27T07:26:10,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:26:10,514 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112766367639368a4b9c9e887d7b74bafa8b_9cd6e5b712012f5d5b13dc2bcef88d1f to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112766367639368a4b9c9e887d7b74bafa8b_9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:26:10,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/cef173a66a2f4522b1b7751aff37a86c, store: [table=TestAcidGuarantees family=A region=9cd6e5b712012f5d5b13dc2bcef88d1f] 2024-11-27T07:26:10,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/cef173a66a2f4522b1b7751aff37a86c is 175, key is test_row_0/A:col10/1732692369484/Put/seqid=0 2024-11-27T07:26:10,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742229_1405 (size=31105) 2024-11-27T07:26:10,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:10,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51540 deadline: 1732692430539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:10,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:10,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:10,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:10,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:10,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51510 deadline: 1732692430549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:10,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:10,553 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:10,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51548 deadline: 1732692430550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:10,553 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:10,553 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:10,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:10,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:10,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:10,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-27T07:26:10,845 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/C/58c3117356904f06b3bc4b9a216f4a6e as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/58c3117356904f06b3bc4b9a216f4a6e 2024-11-27T07:26:10,849 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9cd6e5b712012f5d5b13dc2bcef88d1f/C of 9cd6e5b712012f5d5b13dc2bcef88d1f into 58c3117356904f06b3bc4b9a216f4a6e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:26:10,849 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cd6e5b712012f5d5b13dc2bcef88d1f: 2024-11-27T07:26:10,849 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., storeName=9cd6e5b712012f5d5b13dc2bcef88d1f/C, priority=13, startTime=1732692369958; duration=0sec 2024-11-27T07:26:10,849 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:26:10,849 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cd6e5b712012f5d5b13dc2bcef88d1f:C 2024-11-27T07:26:10,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:10,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51540 deadline: 1732692430849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:10,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:10,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:10,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:10,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:10,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51510 deadline: 1732692430859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:10,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:10,862 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:10,863 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:10,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51548 deadline: 1732692430859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:10,863 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:10,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:10,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:10,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:10,921 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=233, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/cef173a66a2f4522b1b7751aff37a86c 2024-11-27T07:26:10,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/B/5b4923f6efde49d28a41f09947c0d66b is 50, key is test_row_0/B:col10/1732692369484/Put/seqid=0 2024-11-27T07:26:10,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742230_1406 (size=12151) 2024-11-27T07:26:11,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:11,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51586 deadline: 1732692431109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:11,114 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 289954c9202d,42117,1732692259671 for 'test_row_2' in 9cd6e5b712012f5d5b13dc2bcef88d1f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8247 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T07:26:11,115 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:11,115 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:11,115 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:11,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:11,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51584 deadline: 1732692431129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:11,136 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 289954c9202d,42117,1732692259671 for 'test_row_1' in 9cd6e5b712012f5d5b13dc2bcef88d1f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8270 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T07:26:11,137 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:11,137 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:11,137 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:11,331 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/B/5b4923f6efde49d28a41f09947c0d66b 2024-11-27T07:26:11,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/C/c56f1fc0b7764d3989f188593207eb5a is 50, key is test_row_0/C:col10/1732692369484/Put/seqid=0 2024-11-27T07:26:11,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742231_1407 (size=12151) 2024-11-27T07:26:11,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:11,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51540 deadline: 1732692431359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:11,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:11,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:11,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:11,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:11,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51510 deadline: 1732692431368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:11,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:11,375 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:11,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51548 deadline: 1732692431368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:11,375 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:11,375 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:11,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:11,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:11,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:11,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-27T07:26:11,744 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/C/c56f1fc0b7764d3989f188593207eb5a 2024-11-27T07:26:11,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/cef173a66a2f4522b1b7751aff37a86c as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/cef173a66a2f4522b1b7751aff37a86c 2024-11-27T07:26:11,770 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/cef173a66a2f4522b1b7751aff37a86c, entries=150, sequenceid=233, filesize=30.4 K 2024-11-27T07:26:11,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/B/5b4923f6efde49d28a41f09947c0d66b as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/5b4923f6efde49d28a41f09947c0d66b 2024-11-27T07:26:11,779 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/5b4923f6efde49d28a41f09947c0d66b, entries=150, sequenceid=233, filesize=11.9 K 2024-11-27T07:26:11,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/C/c56f1fc0b7764d3989f188593207eb5a as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/c56f1fc0b7764d3989f188593207eb5a 2024-11-27T07:26:11,788 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/c56f1fc0b7764d3989f188593207eb5a, entries=150, sequenceid=233, filesize=11.9 K 2024-11-27T07:26:11,790 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for 9cd6e5b712012f5d5b13dc2bcef88d1f in 1752ms, sequenceid=233, compaction requested=false 2024-11-27T07:26:11,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 9cd6e5b712012f5d5b13dc2bcef88d1f: 2024-11-27T07:26:11,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:11,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-27T07:26:11,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-27T07:26:11,793 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-27T07:26:11,793 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2130 sec 2024-11-27T07:26:11,796 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 2.2240 sec 2024-11-27T07:26:12,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:26:12,384 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T07:26:12,384 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9cd6e5b712012f5d5b13dc2bcef88d1f 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-27T07:26:12,397 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411277327607443f64e4eb0a51df999d82438_9cd6e5b712012f5d5b13dc2bcef88d1f is 50, key is test_row_0/A:col10/1732692372383/Put/seqid=0 2024-11-27T07:26:12,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742232_1408 (size=14844) 2024-11-27T07:26:12,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:12,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51510 deadline: 1732692432418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:12,427 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:12,427 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:12,427 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:12,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:12,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51540 deadline: 1732692432419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:12,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:12,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:12,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:12,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:12,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51548 deadline: 1732692432421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:12,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:12,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:12,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:12,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:12,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51510 deadline: 1732692432529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:12,535 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:12,535 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:12,535 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:12,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:12,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51540 deadline: 1732692432539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:12,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:12,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:12,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:12,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:12,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51548 deadline: 1732692432549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:12,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:12,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:12,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:12,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:12,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51510 deadline: 1732692432739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:12,744 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:12,744 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:12,744 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:12,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:12,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51540 deadline: 1732692432759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:12,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:12,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:12,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51548 deadline: 1732692432759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:12,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:12,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:12,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:12,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:12,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:12,803 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:26:12,807 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411277327607443f64e4eb0a51df999d82438_9cd6e5b712012f5d5b13dc2bcef88d1f to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411277327607443f64e4eb0a51df999d82438_9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:26:12,808 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/991502337a654acab84af6c9a6233852, store: [table=TestAcidGuarantees family=A region=9cd6e5b712012f5d5b13dc2bcef88d1f] 2024-11-27T07:26:12,809 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/991502337a654acab84af6c9a6233852 is 175, key is test_row_0/A:col10/1732692372383/Put/seqid=0 2024-11-27T07:26:12,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742233_1409 (size=39799) 2024-11-27T07:26:13,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:13,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51510 deadline: 1732692433049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:13,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:13,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:13,057 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:13,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:13,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51540 deadline: 1732692433079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:13,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:13,086 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:13,086 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:13,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51548 deadline: 1732692433079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:13,086 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:13,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:13,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:13,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:13,214 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=258, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/991502337a654acab84af6c9a6233852 2024-11-27T07:26:13,222 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/B/c4f1437d0bc84502a766c614ac54c6aa is 50, key is test_row_0/B:col10/1732692372383/Put/seqid=0 2024-11-27T07:26:13,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742234_1410 (size=12201) 2024-11-27T07:26:13,241 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/B/c4f1437d0bc84502a766c614ac54c6aa 2024-11-27T07:26:13,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/C/46d2189b128143cbaba9048439b73351 is 50, key is test_row_0/C:col10/1732692372383/Put/seqid=0 2024-11-27T07:26:13,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742235_1411 (size=12201) 2024-11-27T07:26:13,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:13,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51510 deadline: 1732692433569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:13,576 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:13,576 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:13,576 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:13,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:13,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51540 deadline: 1732692433599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:13,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:13,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:13,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:13,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:13,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51548 deadline: 1732692433599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:13,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:13,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:13,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:13,659 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/C/46d2189b128143cbaba9048439b73351 2024-11-27T07:26:13,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/991502337a654acab84af6c9a6233852 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/991502337a654acab84af6c9a6233852 2024-11-27T07:26:13,665 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/991502337a654acab84af6c9a6233852, entries=200, sequenceid=258, filesize=38.9 K 2024-11-27T07:26:13,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/B/c4f1437d0bc84502a766c614ac54c6aa as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/c4f1437d0bc84502a766c614ac54c6aa 2024-11-27T07:26:13,668 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/c4f1437d0bc84502a766c614ac54c6aa, entries=150, sequenceid=258, filesize=11.9 K 2024-11-27T07:26:13,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/C/46d2189b128143cbaba9048439b73351 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/46d2189b128143cbaba9048439b73351 2024-11-27T07:26:13,671 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/46d2189b128143cbaba9048439b73351, entries=150, sequenceid=258, filesize=11.9 K 2024-11-27T07:26:13,672 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 9cd6e5b712012f5d5b13dc2bcef88d1f in 1288ms, sequenceid=258, compaction requested=true 2024-11-27T07:26:13,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9cd6e5b712012f5d5b13dc2bcef88d1f: 2024-11-27T07:26:13,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cd6e5b712012f5d5b13dc2bcef88d1f:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T07:26:13,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:26:13,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cd6e5b712012f5d5b13dc2bcef88d1f:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T07:26:13,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:26:13,672 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:26:13,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cd6e5b712012f5d5b13dc2bcef88d1f:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T07:26:13,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:26:13,672 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:26:13,673 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37015 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:26:13,673 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102521 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:26:13,673 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 9cd6e5b712012f5d5b13dc2bcef88d1f/B is initiating minor compaction (all files) 2024-11-27T07:26:13,673 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1541): 9cd6e5b712012f5d5b13dc2bcef88d1f/A is initiating minor compaction (all files) 2024-11-27T07:26:13,673 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cd6e5b712012f5d5b13dc2bcef88d1f/B in TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:13,673 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cd6e5b712012f5d5b13dc2bcef88d1f/A in TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:13,673 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/222bb4458c734a6fba1fb95730dec978, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/5b4923f6efde49d28a41f09947c0d66b, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/c4f1437d0bc84502a766c614ac54c6aa] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp, totalSize=36.1 K 2024-11-27T07:26:13,673 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/6b1f7f00b7c84502bc314d8211a2d2e7, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/cef173a66a2f4522b1b7751aff37a86c, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/991502337a654acab84af6c9a6233852] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp, totalSize=100.1 K 2024-11-27T07:26:13,673 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:13,673 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. files: [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/6b1f7f00b7c84502bc314d8211a2d2e7, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/cef173a66a2f4522b1b7751aff37a86c, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/991502337a654acab84af6c9a6233852] 2024-11-27T07:26:13,674 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 222bb4458c734a6fba1fb95730dec978, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1732692368285 2024-11-27T07:26:13,674 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b1f7f00b7c84502bc314d8211a2d2e7, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1732692368285 2024-11-27T07:26:13,674 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b4923f6efde49d28a41f09947c0d66b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732692369468 2024-11-27T07:26:13,674 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting cef173a66a2f4522b1b7751aff37a86c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732692369468 2024-11-27T07:26:13,674 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting c4f1437d0bc84502a766c614ac54c6aa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732692370190 2024-11-27T07:26:13,674 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 991502337a654acab84af6c9a6233852, keycount=200, bloomtype=ROW, size=38.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732692370157 2024-11-27T07:26:13,679 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9cd6e5b712012f5d5b13dc2bcef88d1f] 2024-11-27T07:26:13,681 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cd6e5b712012f5d5b13dc2bcef88d1f#B#compaction#343 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:26:13,681 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127efad511c8fc74616a12288d134906184_9cd6e5b712012f5d5b13dc2bcef88d1f store=[table=TestAcidGuarantees family=A region=9cd6e5b712012f5d5b13dc2bcef88d1f] 2024-11-27T07:26:13,682 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/B/d165a43bcd53488ab141522dada23bdc is 50, key is test_row_0/B:col10/1732692372383/Put/seqid=0 2024-11-27T07:26:13,684 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127efad511c8fc74616a12288d134906184_9cd6e5b712012f5d5b13dc2bcef88d1f, store=[table=TestAcidGuarantees family=A region=9cd6e5b712012f5d5b13dc2bcef88d1f] 2024-11-27T07:26:13,684 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127efad511c8fc74616a12288d134906184_9cd6e5b712012f5d5b13dc2bcef88d1f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9cd6e5b712012f5d5b13dc2bcef88d1f] 2024-11-27T07:26:13,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742236_1412 (size=12815) 2024-11-27T07:26:13,693 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/B/d165a43bcd53488ab141522dada23bdc as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/d165a43bcd53488ab141522dada23bdc 2024-11-27T07:26:13,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742237_1413 (size=4469) 2024-11-27T07:26:13,701 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9cd6e5b712012f5d5b13dc2bcef88d1f/B of 9cd6e5b712012f5d5b13dc2bcef88d1f into d165a43bcd53488ab141522dada23bdc(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:26:13,701 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cd6e5b712012f5d5b13dc2bcef88d1f: 2024-11-27T07:26:13,701 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., storeName=9cd6e5b712012f5d5b13dc2bcef88d1f/B, priority=13, startTime=1732692373672; duration=0sec 2024-11-27T07:26:13,701 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:26:13,701 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cd6e5b712012f5d5b13dc2bcef88d1f:B 2024-11-27T07:26:13,701 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:26:13,701 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cd6e5b712012f5d5b13dc2bcef88d1f#A#compaction#342 average throughput is 1.11 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:26:13,702 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/b7e30a55bf694a4aaf78ae9a23b8a433 is 175, key is test_row_0/A:col10/1732692372383/Put/seqid=0 2024-11-27T07:26:13,702 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37015 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:26:13,702 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 9cd6e5b712012f5d5b13dc2bcef88d1f/C is initiating minor compaction (all files) 2024-11-27T07:26:13,702 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cd6e5b712012f5d5b13dc2bcef88d1f/C in TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:13,702 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/58c3117356904f06b3bc4b9a216f4a6e, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/c56f1fc0b7764d3989f188593207eb5a, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/46d2189b128143cbaba9048439b73351] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp, totalSize=36.1 K 2024-11-27T07:26:13,703 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 58c3117356904f06b3bc4b9a216f4a6e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1732692368285 2024-11-27T07:26:13,704 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting c56f1fc0b7764d3989f188593207eb5a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732692369468 2024-11-27T07:26:13,704 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 46d2189b128143cbaba9048439b73351, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732692370190 2024-11-27T07:26:13,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742238_1414 (size=31769) 2024-11-27T07:26:13,711 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/b7e30a55bf694a4aaf78ae9a23b8a433 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/b7e30a55bf694a4aaf78ae9a23b8a433 2024-11-27T07:26:13,712 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cd6e5b712012f5d5b13dc2bcef88d1f#C#compaction#344 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:26:13,713 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/C/db99f21ab99d432fbfdb622a5f8e040f is 50, key is test_row_0/C:col10/1732692372383/Put/seqid=0 2024-11-27T07:26:13,716 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9cd6e5b712012f5d5b13dc2bcef88d1f/A of 9cd6e5b712012f5d5b13dc2bcef88d1f into b7e30a55bf694a4aaf78ae9a23b8a433(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:26:13,716 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cd6e5b712012f5d5b13dc2bcef88d1f: 2024-11-27T07:26:13,716 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., storeName=9cd6e5b712012f5d5b13dc2bcef88d1f/A, priority=13, startTime=1732692373672; duration=0sec 2024-11-27T07:26:13,716 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:26:13,716 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cd6e5b712012f5d5b13dc2bcef88d1f:A 2024-11-27T07:26:13,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742239_1415 (size=12815) 2024-11-27T07:26:13,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-27T07:26:13,739 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T07:26:13,739 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T07:26:13,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-27T07:26:13,741 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T07:26:13,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-27T07:26:13,741 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T07:26:13,741 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T07:26:13,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-27T07:26:13,893 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-27T07:26:13,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:13,893 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 9cd6e5b712012f5d5b13dc2bcef88d1f 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-27T07:26:13,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127d02469c8ffd145b9b75983d29ed4d9e3_9cd6e5b712012f5d5b13dc2bcef88d1f is 50, key is test_row_0/A:col10/1732692372419/Put/seqid=0 2024-11-27T07:26:13,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742240_1416 (size=12454) 2024-11-27T07:26:14,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-27T07:26:14,123 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/C/db99f21ab99d432fbfdb622a5f8e040f as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/db99f21ab99d432fbfdb622a5f8e040f 2024-11-27T07:26:14,127 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9cd6e5b712012f5d5b13dc2bcef88d1f/C of 9cd6e5b712012f5d5b13dc2bcef88d1f into db99f21ab99d432fbfdb622a5f8e040f(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:26:14,127 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cd6e5b712012f5d5b13dc2bcef88d1f: 2024-11-27T07:26:14,127 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., storeName=9cd6e5b712012f5d5b13dc2bcef88d1f/C, priority=13, startTime=1732692373672; duration=0sec 2024-11-27T07:26:14,128 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:26:14,128 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cd6e5b712012f5d5b13dc2bcef88d1f:C 2024-11-27T07:26:14,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:26:14,306 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127d02469c8ffd145b9b75983d29ed4d9e3_9cd6e5b712012f5d5b13dc2bcef88d1f to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127d02469c8ffd145b9b75983d29ed4d9e3_9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:26:14,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/dd8e3fb5246d41c5a4f9ad0ce5f97c25, store: [table=TestAcidGuarantees family=A region=9cd6e5b712012f5d5b13dc2bcef88d1f] 2024-11-27T07:26:14,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/dd8e3fb5246d41c5a4f9ad0ce5f97c25 is 175, key is test_row_0/A:col10/1732692372419/Put/seqid=0 2024-11-27T07:26:14,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742241_1417 (size=31255) 2024-11-27T07:26:14,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-27T07:26:14,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. as already flushing 2024-11-27T07:26:14,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:26:14,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:14,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51510 deadline: 1732692434654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:14,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:14,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:14,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:14,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:14,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51548 deadline: 1732692434662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:14,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:14,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:14,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51540 deadline: 1732692434662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:14,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:14,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:14,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:14,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:14,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:14,711 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=274, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/dd8e3fb5246d41c5a4f9ad0ce5f97c25 2024-11-27T07:26:14,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/B/b0dcb384345d41b9bbf5d0895c3a66a6 is 50, key is test_row_0/B:col10/1732692372419/Put/seqid=0 2024-11-27T07:26:14,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742242_1418 (size=12301) 2024-11-27T07:26:14,729 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/B/b0dcb384345d41b9bbf5d0895c3a66a6 2024-11-27T07:26:14,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/C/9e339863b2b844ba96592008da221246 is 50, key is test_row_0/C:col10/1732692372419/Put/seqid=0 2024-11-27T07:26:14,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742243_1419 (size=12301) 2024-11-27T07:26:14,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:14,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51510 deadline: 1732692434769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:14,774 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:14,774 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:14,774 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:14,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:14,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51548 deadline: 1732692434779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:14,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:14,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:14,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:14,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:14,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51540 deadline: 1732692434779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:14,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:14,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:14,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:14,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-27T07:26:14,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:14,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51510 deadline: 1732692434980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:14,987 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:14,987 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:14,987 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:14,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:14,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51548 deadline: 1732692434989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:14,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:14,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:14,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:14,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:14,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51540 deadline: 1732692434989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:14,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:14,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:14,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:15,149 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/C/9e339863b2b844ba96592008da221246 2024-11-27T07:26:15,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/dd8e3fb5246d41c5a4f9ad0ce5f97c25 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/dd8e3fb5246d41c5a4f9ad0ce5f97c25 2024-11-27T07:26:15,157 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/dd8e3fb5246d41c5a4f9ad0ce5f97c25, entries=150, sequenceid=274, filesize=30.5 K 2024-11-27T07:26:15,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/B/b0dcb384345d41b9bbf5d0895c3a66a6 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/b0dcb384345d41b9bbf5d0895c3a66a6 2024-11-27T07:26:15,161 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/b0dcb384345d41b9bbf5d0895c3a66a6, entries=150, sequenceid=274, filesize=12.0 K 2024-11-27T07:26:15,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/C/9e339863b2b844ba96592008da221246 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/9e339863b2b844ba96592008da221246 2024-11-27T07:26:15,165 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/9e339863b2b844ba96592008da221246, entries=150, sequenceid=274, filesize=12.0 K 2024-11-27T07:26:15,166 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 9cd6e5b712012f5d5b13dc2bcef88d1f in 1273ms, sequenceid=274, compaction requested=false 2024-11-27T07:26:15,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 9cd6e5b712012f5d5b13dc2bcef88d1f: 2024-11-27T07:26:15,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:15,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-27T07:26:15,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-27T07:26:15,170 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-27T07:26:15,170 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4270 sec 2024-11-27T07:26:15,171 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 1.4310 sec 2024-11-27T07:26:15,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:26:15,294 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T07:26:15,294 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9cd6e5b712012f5d5b13dc2bcef88d1f 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-27T07:26:15,300 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127a7128ca2f9a2463088c2636694a631d7_9cd6e5b712012f5d5b13dc2bcef88d1f is 50, key is test_row_0/A:col10/1732692374661/Put/seqid=0 2024-11-27T07:26:15,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742244_1420 (size=14994) 2024-11-27T07:26:15,305 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:26:15,308 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127a7128ca2f9a2463088c2636694a631d7_9cd6e5b712012f5d5b13dc2bcef88d1f to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127a7128ca2f9a2463088c2636694a631d7_9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:26:15,308 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/400e9093fb9243f293e92f45b864d8c0, store: [table=TestAcidGuarantees family=A region=9cd6e5b712012f5d5b13dc2bcef88d1f] 2024-11-27T07:26:15,309 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/400e9093fb9243f293e92f45b864d8c0 is 175, key is test_row_0/A:col10/1732692374661/Put/seqid=0 2024-11-27T07:26:15,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742245_1421 (size=39949) 2024-11-27T07:26:15,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:15,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51540 deadline: 1732692435327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:15,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:15,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:15,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:15,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:15,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51548 deadline: 1732692435328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:15,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:15,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:15,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:15,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:15,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51510 deadline: 1732692435329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:15,339 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:15,339 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:15,339 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:15,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:15,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51540 deadline: 1732692435439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:15,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:15,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:15,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:15,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:15,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51548 deadline: 1732692435439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:15,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:15,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:15,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:15,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:15,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51510 deadline: 1732692435448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:15,459 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:15,459 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:15,459 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:15,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:15,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51540 deadline: 1732692435659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:15,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:15,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:15,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:15,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:15,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51548 deadline: 1732692435659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:15,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:15,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:15,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:15,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:15,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51510 deadline: 1732692435669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:15,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:15,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:15,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:15,712 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=298, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/400e9093fb9243f293e92f45b864d8c0 2024-11-27T07:26:15,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/B/798970f065bd480c812f4093419ced10 is 50, key is test_row_0/B:col10/1732692374661/Put/seqid=0 2024-11-27T07:26:15,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742246_1422 (size=12301) 2024-11-27T07:26:15,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-27T07:26:15,889 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T07:26:15,890 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T07:26:15,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-27T07:26:15,892 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T07:26:15,893 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T07:26:15,893 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T07:26:15,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-27T07:26:15,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:15,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51548 deadline: 1732692435969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:15,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:15,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:15,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:15,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:15,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51540 deadline: 1732692435979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:15,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:15,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51510 deadline: 1732692435979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:15,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:15,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:15,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:15,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:15,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:15,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:15,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-27T07:26:16,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-27T07:26:16,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:16,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. as already flushing 2024-11-27T07:26:16,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:16,046 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:16,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:16,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:16,122 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/B/798970f065bd480c812f4093419ced10 2024-11-27T07:26:16,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/C/a1617fd9006e4fe79016e8ec8ff15d7d is 50, key is test_row_0/C:col10/1732692374661/Put/seqid=0 2024-11-27T07:26:16,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742247_1423 (size=12301) 2024-11-27T07:26:16,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-27T07:26:16,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:16,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. as already flushing 2024-11-27T07:26:16,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:16,198 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:16,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:16,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:16,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-27T07:26:16,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-27T07:26:16,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:16,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. as already flushing 2024-11-27T07:26:16,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:16,351 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:16,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:16,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:16,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:16,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51548 deadline: 1732692436479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:16,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:16,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:16,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:16,503 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-27T07:26:16,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:16,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. as already flushing 2024-11-27T07:26:16,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:16,504 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:16,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:16,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:16,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:16,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51540 deadline: 1732692436499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:16,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:16,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:16,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:16,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:16,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51510 deadline: 1732692436499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:16,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:16,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:16,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:16,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-27T07:26:16,536 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/C/a1617fd9006e4fe79016e8ec8ff15d7d 2024-11-27T07:26:16,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/400e9093fb9243f293e92f45b864d8c0 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/400e9093fb9243f293e92f45b864d8c0 2024-11-27T07:26:16,542 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/400e9093fb9243f293e92f45b864d8c0, entries=200, sequenceid=298, filesize=39.0 K 2024-11-27T07:26:16,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/B/798970f065bd480c812f4093419ced10 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/798970f065bd480c812f4093419ced10 2024-11-27T07:26:16,545 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/798970f065bd480c812f4093419ced10, entries=150, sequenceid=298, filesize=12.0 K 2024-11-27T07:26:16,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/C/a1617fd9006e4fe79016e8ec8ff15d7d as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/a1617fd9006e4fe79016e8ec8ff15d7d 2024-11-27T07:26:16,549 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/a1617fd9006e4fe79016e8ec8ff15d7d, entries=150, sequenceid=298, filesize=12.0 K 2024-11-27T07:26:16,549 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 9cd6e5b712012f5d5b13dc2bcef88d1f in 1255ms, sequenceid=298, compaction requested=true 2024-11-27T07:26:16,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9cd6e5b712012f5d5b13dc2bcef88d1f: 2024-11-27T07:26:16,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cd6e5b712012f5d5b13dc2bcef88d1f:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T07:26:16,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:26:16,550 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:26:16,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cd6e5b712012f5d5b13dc2bcef88d1f:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T07:26:16,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:26:16,550 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:26:16,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cd6e5b712012f5d5b13dc2bcef88d1f:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T07:26:16,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:26:16,550 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102973 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:26:16,550 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37417 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:26:16,550 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 9cd6e5b712012f5d5b13dc2bcef88d1f/B is initiating minor compaction (all files) 2024-11-27T07:26:16,550 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1541): 9cd6e5b712012f5d5b13dc2bcef88d1f/A is initiating minor compaction (all files) 2024-11-27T07:26:16,550 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cd6e5b712012f5d5b13dc2bcef88d1f/B in TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:16,550 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cd6e5b712012f5d5b13dc2bcef88d1f/A in TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:16,551 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/d165a43bcd53488ab141522dada23bdc, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/b0dcb384345d41b9bbf5d0895c3a66a6, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/798970f065bd480c812f4093419ced10] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp, totalSize=36.5 K 2024-11-27T07:26:16,551 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/b7e30a55bf694a4aaf78ae9a23b8a433, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/dd8e3fb5246d41c5a4f9ad0ce5f97c25, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/400e9093fb9243f293e92f45b864d8c0] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp, totalSize=100.6 K 2024-11-27T07:26:16,551 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:16,551 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. files: [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/b7e30a55bf694a4aaf78ae9a23b8a433, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/dd8e3fb5246d41c5a4f9ad0ce5f97c25, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/400e9093fb9243f293e92f45b864d8c0] 2024-11-27T07:26:16,551 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting d165a43bcd53488ab141522dada23bdc, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732692370190 2024-11-27T07:26:16,551 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7e30a55bf694a4aaf78ae9a23b8a433, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732692370190 2024-11-27T07:26:16,551 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting b0dcb384345d41b9bbf5d0895c3a66a6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732692372406 2024-11-27T07:26:16,551 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd8e3fb5246d41c5a4f9ad0ce5f97c25, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732692372406 2024-11-27T07:26:16,552 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 798970f065bd480c812f4093419ced10, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1732692374645 2024-11-27T07:26:16,552 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 400e9093fb9243f293e92f45b864d8c0, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1732692374645 2024-11-27T07:26:16,556 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9cd6e5b712012f5d5b13dc2bcef88d1f] 2024-11-27T07:26:16,558 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cd6e5b712012f5d5b13dc2bcef88d1f#B#compaction#352 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:26:16,559 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/B/961618df80674356b95f431bc60743be is 50, key is test_row_0/B:col10/1732692374661/Put/seqid=0 2024-11-27T07:26:16,560 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127d65b6b71b0eb4dad8ed51c15c4f9e8cb_9cd6e5b712012f5d5b13dc2bcef88d1f store=[table=TestAcidGuarantees family=A region=9cd6e5b712012f5d5b13dc2bcef88d1f] 2024-11-27T07:26:16,562 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127d65b6b71b0eb4dad8ed51c15c4f9e8cb_9cd6e5b712012f5d5b13dc2bcef88d1f, store=[table=TestAcidGuarantees family=A region=9cd6e5b712012f5d5b13dc2bcef88d1f] 2024-11-27T07:26:16,562 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127d65b6b71b0eb4dad8ed51c15c4f9e8cb_9cd6e5b712012f5d5b13dc2bcef88d1f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9cd6e5b712012f5d5b13dc2bcef88d1f] 2024-11-27T07:26:16,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742248_1424 (size=13017) 2024-11-27T07:26:16,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742249_1425 (size=4469) 2024-11-27T07:26:16,575 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/B/961618df80674356b95f431bc60743be as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/961618df80674356b95f431bc60743be 2024-11-27T07:26:16,580 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9cd6e5b712012f5d5b13dc2bcef88d1f/B of 9cd6e5b712012f5d5b13dc2bcef88d1f into 961618df80674356b95f431bc60743be(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:26:16,580 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cd6e5b712012f5d5b13dc2bcef88d1f: 2024-11-27T07:26:16,580 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., storeName=9cd6e5b712012f5d5b13dc2bcef88d1f/B, priority=13, startTime=1732692376550; duration=0sec 2024-11-27T07:26:16,580 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:26:16,580 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cd6e5b712012f5d5b13dc2bcef88d1f:B 2024-11-27T07:26:16,580 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:26:16,581 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37417 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:26:16,582 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 9cd6e5b712012f5d5b13dc2bcef88d1f/C is initiating minor compaction (all files) 2024-11-27T07:26:16,582 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cd6e5b712012f5d5b13dc2bcef88d1f/C in TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:16,582 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/db99f21ab99d432fbfdb622a5f8e040f, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/9e339863b2b844ba96592008da221246, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/a1617fd9006e4fe79016e8ec8ff15d7d] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp, totalSize=36.5 K 2024-11-27T07:26:16,582 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting db99f21ab99d432fbfdb622a5f8e040f, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732692370190 2024-11-27T07:26:16,584 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e339863b2b844ba96592008da221246, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732692372406 2024-11-27T07:26:16,584 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting a1617fd9006e4fe79016e8ec8ff15d7d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1732692374645 2024-11-27T07:26:16,591 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cd6e5b712012f5d5b13dc2bcef88d1f#C#compaction#353 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:26:16,591 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/C/529225bfc6c0412eb8cca71d18417cba is 50, key is test_row_0/C:col10/1732692374661/Put/seqid=0 2024-11-27T07:26:16,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742250_1426 (size=13017) 2024-11-27T07:26:16,656 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-27T07:26:16,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:16,656 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 9cd6e5b712012f5d5b13dc2bcef88d1f 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-27T07:26:16,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411272330fc70083c4441ac8f619aa7419081_9cd6e5b712012f5d5b13dc2bcef88d1f is 50, key is test_row_0/A:col10/1732692375328/Put/seqid=0 2024-11-27T07:26:16,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742251_1427 (size=12454) 2024-11-27T07:26:16,805 INFO [Thread-1558 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1558. 2024-11-27T07:26:16,805 DEBUG [Thread-1558 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T07:26:16,805 DEBUG [Thread-1558 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:26:16,805 DEBUG [Thread-1558 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:26:16,805 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T07:26:16,807 INFO [Thread-1556 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1556. 2024-11-27T07:26:16,807 DEBUG [Thread-1556 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T07:26:16,807 DEBUG [Thread-1556 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:26:16,807 DEBUG [Thread-1556 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:26:16,807 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T07:26:16,810 INFO [Thread-1562 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1562. 2024-11-27T07:26:16,810 DEBUG [Thread-1562 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T07:26:16,810 DEBUG [Thread-1562 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:26:16,810 DEBUG [Thread-1562 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:26:16,810 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T07:26:16,811 INFO [Thread-1560 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1560. 2024-11-27T07:26:16,811 DEBUG [Thread-1560 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T07:26:16,811 DEBUG [Thread-1560 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:26:16,811 DEBUG [Thread-1560 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:26:16,811 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T07:26:16,814 INFO [Thread-1564 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1564. 2024-11-27T07:26:16,814 DEBUG [Thread-1564 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T07:26:16,814 DEBUG [Thread-1564 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:26:16,814 DEBUG [Thread-1564 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:26:16,814 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T07:26:16,973 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cd6e5b712012f5d5b13dc2bcef88d1f#A#compaction#351 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:26:16,973 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/1c78208700f24db6bd058a7dfc100b67 is 175, key is test_row_0/A:col10/1732692374661/Put/seqid=0 2024-11-27T07:26:16,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742252_1428 (size=31971) 2024-11-27T07:26:17,006 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/C/529225bfc6c0412eb8cca71d18417cba as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/529225bfc6c0412eb8cca71d18417cba 2024-11-27T07:26:17,012 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9cd6e5b712012f5d5b13dc2bcef88d1f/C of 9cd6e5b712012f5d5b13dc2bcef88d1f into 529225bfc6c0412eb8cca71d18417cba(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:26:17,012 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cd6e5b712012f5d5b13dc2bcef88d1f: 2024-11-27T07:26:17,012 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., storeName=9cd6e5b712012f5d5b13dc2bcef88d1f/C, priority=13, startTime=1732692376550; duration=0sec 2024-11-27T07:26:17,013 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:26:17,013 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cd6e5b712012f5d5b13dc2bcef88d1f:C 2024-11-27T07:26:17,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-27T07:26:17,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:26:17,073 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411272330fc70083c4441ac8f619aa7419081_9cd6e5b712012f5d5b13dc2bcef88d1f to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411272330fc70083c4441ac8f619aa7419081_9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:26:17,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/621a8a21d6264e8baa781e262b87d39f, store: [table=TestAcidGuarantees family=A region=9cd6e5b712012f5d5b13dc2bcef88d1f] 2024-11-27T07:26:17,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/621a8a21d6264e8baa781e262b87d39f is 175, key is test_row_0/A:col10/1732692375328/Put/seqid=0 2024-11-27T07:26:17,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742253_1429 (size=31255) 2024-11-27T07:26:17,382 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/1c78208700f24db6bd058a7dfc100b67 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/1c78208700f24db6bd058a7dfc100b67 2024-11-27T07:26:17,386 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9cd6e5b712012f5d5b13dc2bcef88d1f/A of 9cd6e5b712012f5d5b13dc2bcef88d1f into 1c78208700f24db6bd058a7dfc100b67(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:26:17,386 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cd6e5b712012f5d5b13dc2bcef88d1f: 2024-11-27T07:26:17,386 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., storeName=9cd6e5b712012f5d5b13dc2bcef88d1f/A, priority=13, startTime=1732692376550; duration=0sec 2024-11-27T07:26:17,386 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:26:17,386 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cd6e5b712012f5d5b13dc2bcef88d1f:A 2024-11-27T07:26:17,481 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=312, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/621a8a21d6264e8baa781e262b87d39f 2024-11-27T07:26:17,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/B/0676e09349b344a1bcf9b7160ee7f8b8 is 50, key is test_row_0/B:col10/1732692375328/Put/seqid=0 2024-11-27T07:26:17,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:26:17,490 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. as already flushing 2024-11-27T07:26:17,491 INFO [Thread-1549 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1549. 2024-11-27T07:26:17,491 DEBUG [Thread-1549 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T07:26:17,491 DEBUG [Thread-1549 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:26:17,491 DEBUG [Thread-1549 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:26:17,491 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T07:26:17,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742254_1430 (size=12301) 2024-11-27T07:26:17,520 INFO [Thread-1547 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1547. 2024-11-27T07:26:17,520 INFO [Thread-1545 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1545. 2024-11-27T07:26:17,520 DEBUG [Thread-1547 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T07:26:17,520 DEBUG [Thread-1545 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T07:26:17,520 DEBUG [Thread-1547 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:26:17,520 DEBUG [Thread-1545 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:26:17,520 DEBUG [Thread-1547 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:26:17,520 DEBUG [Thread-1545 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:26:17,520 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T07:26:17,520 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T07:26:17,892 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/B/0676e09349b344a1bcf9b7160ee7f8b8 2024-11-27T07:26:17,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/C/55db2297a2494427be1c04b36f59db35 is 50, key is test_row_0/C:col10/1732692375328/Put/seqid=0 2024-11-27T07:26:17,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742255_1431 (size=12301) 2024-11-27T07:26:17,970 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-27T07:26:18,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-27T07:26:18,307 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/C/55db2297a2494427be1c04b36f59db35 2024-11-27T07:26:18,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/621a8a21d6264e8baa781e262b87d39f as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/621a8a21d6264e8baa781e262b87d39f 2024-11-27T07:26:18,319 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/621a8a21d6264e8baa781e262b87d39f, entries=150, sequenceid=312, filesize=30.5 K 2024-11-27T07:26:18,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/B/0676e09349b344a1bcf9b7160ee7f8b8 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/0676e09349b344a1bcf9b7160ee7f8b8 2024-11-27T07:26:18,322 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/0676e09349b344a1bcf9b7160ee7f8b8, entries=150, sequenceid=312, filesize=12.0 K 2024-11-27T07:26:18,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/C/55db2297a2494427be1c04b36f59db35 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/55db2297a2494427be1c04b36f59db35 2024-11-27T07:26:18,325 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/55db2297a2494427be1c04b36f59db35, entries=150, sequenceid=312, filesize=12.0 K 2024-11-27T07:26:18,326 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=20.13 KB/20610 for 9cd6e5b712012f5d5b13dc2bcef88d1f in 1670ms, sequenceid=312, compaction requested=false 2024-11-27T07:26:18,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 9cd6e5b712012f5d5b13dc2bcef88d1f: 2024-11-27T07:26:18,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:18,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-27T07:26:18,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-27T07:26:18,329 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-27T07:26:18,329 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4340 sec 2024-11-27T07:26:18,330 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 2.4390 sec 2024-11-27T07:26:20,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-27T07:26:20,050 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T07:26:21,151 INFO [Thread-1553 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1553. 2024-11-27T07:26:21,151 DEBUG [Thread-1553 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T07:26:21,151 DEBUG [Thread-1553 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:26:21,151 DEBUG [Thread-1553 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:26:21,151 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T07:26:21,213 INFO [Thread-1551 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1551. 2024-11-27T07:26:21,213 DEBUG [Thread-1551 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T07:26:21,213 DEBUG [Thread-1551 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:26:21,213 DEBUG [Thread-1551 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:26:21,213 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T07:26:21,213 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-27T07:26:21,213 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-11-27T07:26:21,213 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 111 2024-11-27T07:26:21,213 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 45 2024-11-27T07:26:21,213 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 18 2024-11-27T07:26:21,213 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 18 2024-11-27T07:26:21,213 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-27T07:26:21,213 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-27T07:26:21,213 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1309 2024-11-27T07:26:21,213 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3925 rows 2024-11-27T07:26:21,213 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1297 2024-11-27T07:26:21,213 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3890 rows 2024-11-27T07:26:21,213 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1306 2024-11-27T07:26:21,213 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3913 rows 2024-11-27T07:26:21,213 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1298 2024-11-27T07:26:21,213 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3890 rows 2024-11-27T07:26:21,213 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1298 2024-11-27T07:26:21,213 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3892 rows 2024-11-27T07:26:21,214 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T07:26:21,214 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T07:26:21,214 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:26:21,214 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:26:21,214 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T07:26:21,215 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-27T07:26:21,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-27T07:26:21,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-27T07:26:21,218 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732692381217"}]},"ts":"1732692381217"} 2024-11-27T07:26:21,219 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-27T07:26:21,219 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-27T07:26:21,220 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-27T07:26:21,221 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9cd6e5b712012f5d5b13dc2bcef88d1f, UNASSIGN}] 2024-11-27T07:26:21,222 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9cd6e5b712012f5d5b13dc2bcef88d1f, UNASSIGN 2024-11-27T07:26:21,222 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=9cd6e5b712012f5d5b13dc2bcef88d1f, regionState=CLOSING, regionLocation=289954c9202d,42117,1732692259671 2024-11-27T07:26:21,223 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43057 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=289954c9202d,42117,1732692259671, table=TestAcidGuarantees, region=9cd6e5b712012f5d5b13dc2bcef88d1f. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-27T07:26:21,224 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9cd6e5b712012f5d5b13dc2bcef88d1f, UNASSIGN because future has completed 2024-11-27T07:26:21,224 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T07:26:21,224 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671}] 2024-11-27T07:26:21,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-27T07:26:21,376 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(122): Close 9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:26:21,376 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T07:26:21,376 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1722): Closing 9cd6e5b712012f5d5b13dc2bcef88d1f, disabling compactions & flushes 2024-11-27T07:26:21,376 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:21,376 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:21,376 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. after waiting 0 ms 2024-11-27T07:26:21,376 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:21,376 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(2902): Flushing 9cd6e5b712012f5d5b13dc2bcef88d1f 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-27T07:26:21,381 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127fe81ab30692c4e499a1525861a1c9159_9cd6e5b712012f5d5b13dc2bcef88d1f is 50, key is test_row_0/A:col10/1732692377519/Put/seqid=0 2024-11-27T07:26:21,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742256_1432 (size=12454) 2024-11-27T07:26:21,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-27T07:26:21,785 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:26:21,788 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127fe81ab30692c4e499a1525861a1c9159_9cd6e5b712012f5d5b13dc2bcef88d1f to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127fe81ab30692c4e499a1525861a1c9159_9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:26:21,788 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/249208ca0dd74e85bb0b1bd86d940b08, store: [table=TestAcidGuarantees family=A region=9cd6e5b712012f5d5b13dc2bcef88d1f] 2024-11-27T07:26:21,789 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/249208ca0dd74e85bb0b1bd86d940b08 is 175, key is test_row_0/A:col10/1732692377519/Put/seqid=0 2024-11-27T07:26:21,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742257_1433 (size=31255) 2024-11-27T07:26:21,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-27T07:26:21,980 WARN [regionserver/289954c9202d:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1 2024-11-27T07:26:21,981 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/289954c9202d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/79bbc270cd324089a506e299686ae483, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/7024a54e1de54ada9936496436bb1e3c, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/60d9b84bdaa04f939dc621f0178a3111, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/db34162c92a84b46bace782fe7821190, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/6068ebc37dd64ae0a7d83c0a19f239ac, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/431d956281d84a719f74a1c4da5c0b87, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/abd923c0d5ac4a0e98edff57a47689f9, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/6a6a3fd18132467ea5a621412a82cfba, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/59984b5eadfb43a08b913fa21ad301aa, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/9a6f3fe690ef480187450b400a823e1e, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/f182720f868b40d1863db13ed6257f5c, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/b3f0410d1ddb488ca6cb753ca8f4643a, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/7c148dff92d148eba5876c2f76497948, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/cd39876837ca4544ad3f01a3d9d9d856, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/d459f223b77742688e098b03db0ef9b7, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/6b1f7f00b7c84502bc314d8211a2d2e7, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/cef173a66a2f4522b1b7751aff37a86c, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/991502337a654acab84af6c9a6233852, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/b7e30a55bf694a4aaf78ae9a23b8a433, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/dd8e3fb5246d41c5a4f9ad0ce5f97c25, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/400e9093fb9243f293e92f45b864d8c0] to archive 2024-11-27T07:26:21,982 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/289954c9202d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T07:26:21,984 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/289954c9202d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/79bbc270cd324089a506e299686ae483 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/79bbc270cd324089a506e299686ae483 2024-11-27T07:26:21,985 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/289954c9202d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/7024a54e1de54ada9936496436bb1e3c to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/7024a54e1de54ada9936496436bb1e3c 2024-11-27T07:26:21,986 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/289954c9202d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/60d9b84bdaa04f939dc621f0178a3111 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/60d9b84bdaa04f939dc621f0178a3111 2024-11-27T07:26:21,987 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/289954c9202d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/db34162c92a84b46bace782fe7821190 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/db34162c92a84b46bace782fe7821190 2024-11-27T07:26:21,988 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/289954c9202d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/6068ebc37dd64ae0a7d83c0a19f239ac to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/6068ebc37dd64ae0a7d83c0a19f239ac 2024-11-27T07:26:21,989 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/289954c9202d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/431d956281d84a719f74a1c4da5c0b87 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/431d956281d84a719f74a1c4da5c0b87 2024-11-27T07:26:21,990 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/289954c9202d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/abd923c0d5ac4a0e98edff57a47689f9 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/abd923c0d5ac4a0e98edff57a47689f9 2024-11-27T07:26:21,991 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/289954c9202d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/6a6a3fd18132467ea5a621412a82cfba to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/6a6a3fd18132467ea5a621412a82cfba 2024-11-27T07:26:21,992 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/289954c9202d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/59984b5eadfb43a08b913fa21ad301aa to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/59984b5eadfb43a08b913fa21ad301aa 2024-11-27T07:26:21,993 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/289954c9202d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/9a6f3fe690ef480187450b400a823e1e to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/9a6f3fe690ef480187450b400a823e1e 2024-11-27T07:26:21,994 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/289954c9202d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/f182720f868b40d1863db13ed6257f5c to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/f182720f868b40d1863db13ed6257f5c 2024-11-27T07:26:21,995 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/289954c9202d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/b3f0410d1ddb488ca6cb753ca8f4643a to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/b3f0410d1ddb488ca6cb753ca8f4643a 2024-11-27T07:26:21,996 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/289954c9202d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/7c148dff92d148eba5876c2f76497948 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/7c148dff92d148eba5876c2f76497948 2024-11-27T07:26:21,997 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/289954c9202d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/cd39876837ca4544ad3f01a3d9d9d856 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/cd39876837ca4544ad3f01a3d9d9d856 2024-11-27T07:26:21,998 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/289954c9202d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/d459f223b77742688e098b03db0ef9b7 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/d459f223b77742688e098b03db0ef9b7 2024-11-27T07:26:21,999 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/289954c9202d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/6b1f7f00b7c84502bc314d8211a2d2e7 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/6b1f7f00b7c84502bc314d8211a2d2e7 2024-11-27T07:26:22,000 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/289954c9202d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/cef173a66a2f4522b1b7751aff37a86c to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/cef173a66a2f4522b1b7751aff37a86c 2024-11-27T07:26:22,001 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/289954c9202d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/991502337a654acab84af6c9a6233852 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/991502337a654acab84af6c9a6233852 2024-11-27T07:26:22,002 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/289954c9202d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/b7e30a55bf694a4aaf78ae9a23b8a433 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/b7e30a55bf694a4aaf78ae9a23b8a433 2024-11-27T07:26:22,003 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/289954c9202d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/dd8e3fb5246d41c5a4f9ad0ce5f97c25 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/dd8e3fb5246d41c5a4f9ad0ce5f97c25 2024-11-27T07:26:22,004 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/289954c9202d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/400e9093fb9243f293e92f45b864d8c0 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/400e9093fb9243f293e92f45b864d8c0 2024-11-27T07:26:22,008 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/289954c9202d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/2755fe75609f42c1bce7d645c75fd5ee, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/fe1a11116b944dc6b8440584cf8bd1d3, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/a0ba446e16694485b6d70e0500c533eb, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/a538e2170a3644a3ae706a5369dfceb6, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/df66a6e50cda4480bb0f9833ef900020, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/55689899011c4000a68c05ce335dbaa3, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/6a3361f5a3b44c568bfdc171b42326e0, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/73dd4eba32d94662938a18ea92c810e4, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/f60b55186b49468eb95e865163518312, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/de0511d5545344ecb3d968e50f363323, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/83d1fa12abe247d4b9862bab9d3e695c, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/e1f4ab66aa934ef79b97754385f15b18, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/c44723700fc443e9b60b32204c16e7ab, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/89bfc11864da4ed391e93b8e72c0683e, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/222bb4458c734a6fba1fb95730dec978, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/5f3b9207350a49a3aa515aacd6429753, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/5b4923f6efde49d28a41f09947c0d66b, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/d165a43bcd53488ab141522dada23bdc, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/c4f1437d0bc84502a766c614ac54c6aa, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/b0dcb384345d41b9bbf5d0895c3a66a6, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/798970f065bd480c812f4093419ced10] to archive 2024-11-27T07:26:22,009 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/289954c9202d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T07:26:22,010 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/289954c9202d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/2755fe75609f42c1bce7d645c75fd5ee to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/2755fe75609f42c1bce7d645c75fd5ee 2024-11-27T07:26:22,011 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/289954c9202d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/fe1a11116b944dc6b8440584cf8bd1d3 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/fe1a11116b944dc6b8440584cf8bd1d3 2024-11-27T07:26:22,012 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/289954c9202d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/a0ba446e16694485b6d70e0500c533eb to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/a0ba446e16694485b6d70e0500c533eb 2024-11-27T07:26:22,013 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/289954c9202d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/a538e2170a3644a3ae706a5369dfceb6 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/a538e2170a3644a3ae706a5369dfceb6 2024-11-27T07:26:22,014 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/289954c9202d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/df66a6e50cda4480bb0f9833ef900020 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/df66a6e50cda4480bb0f9833ef900020 2024-11-27T07:26:22,015 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/289954c9202d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/55689899011c4000a68c05ce335dbaa3 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/55689899011c4000a68c05ce335dbaa3 2024-11-27T07:26:22,016 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/289954c9202d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/6a3361f5a3b44c568bfdc171b42326e0 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/6a3361f5a3b44c568bfdc171b42326e0 2024-11-27T07:26:22,017 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/289954c9202d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/73dd4eba32d94662938a18ea92c810e4 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/73dd4eba32d94662938a18ea92c810e4 2024-11-27T07:26:22,018 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/289954c9202d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/f60b55186b49468eb95e865163518312 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/f60b55186b49468eb95e865163518312 2024-11-27T07:26:22,019 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/289954c9202d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/de0511d5545344ecb3d968e50f363323 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/de0511d5545344ecb3d968e50f363323 2024-11-27T07:26:22,020 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/289954c9202d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/83d1fa12abe247d4b9862bab9d3e695c to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/83d1fa12abe247d4b9862bab9d3e695c 2024-11-27T07:26:22,020 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/289954c9202d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/e1f4ab66aa934ef79b97754385f15b18 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/e1f4ab66aa934ef79b97754385f15b18 2024-11-27T07:26:22,021 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/289954c9202d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/c44723700fc443e9b60b32204c16e7ab to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/c44723700fc443e9b60b32204c16e7ab 2024-11-27T07:26:22,022 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/289954c9202d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/89bfc11864da4ed391e93b8e72c0683e to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/89bfc11864da4ed391e93b8e72c0683e 2024-11-27T07:26:22,022 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/289954c9202d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/222bb4458c734a6fba1fb95730dec978 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/222bb4458c734a6fba1fb95730dec978 2024-11-27T07:26:22,023 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/289954c9202d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/5f3b9207350a49a3aa515aacd6429753 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/5f3b9207350a49a3aa515aacd6429753 2024-11-27T07:26:22,024 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/289954c9202d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/5b4923f6efde49d28a41f09947c0d66b to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/5b4923f6efde49d28a41f09947c0d66b 2024-11-27T07:26:22,025 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/289954c9202d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/d165a43bcd53488ab141522dada23bdc to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/d165a43bcd53488ab141522dada23bdc 2024-11-27T07:26:22,025 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/289954c9202d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/c4f1437d0bc84502a766c614ac54c6aa to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/c4f1437d0bc84502a766c614ac54c6aa 2024-11-27T07:26:22,026 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/289954c9202d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/b0dcb384345d41b9bbf5d0895c3a66a6 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/b0dcb384345d41b9bbf5d0895c3a66a6 2024-11-27T07:26:22,027 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/289954c9202d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/798970f065bd480c812f4093419ced10 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/798970f065bd480c812f4093419ced10 2024-11-27T07:26:22,029 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/289954c9202d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/3fafbde684bd455d80f2725242a1eb07, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/b0a1adabfb6841bdaaf4d2a4581f0040, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/498477fe7a9548daa93c462a2f24b842, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/e282058b049a460b83fa3ec126537667, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/1b65b08c5b0c4d63af85f743429f075a, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/6b0dece1caba4c7bb71ae164750bce10, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/0e378feb740c4c878a496a4d488e4bc5, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/a11c8c23870040b29c0d36e675d3bd6e, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/180862389c0c4704a09325d111a11770, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/38be5e554bcb4481bcf642b9af3e8613, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/357dc42b39c048c6a42c3c684d7499db, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/8f4d1b717d9d46f98d1bde8ec649ffe9, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/681ccb68fa3c4f4fb56b8204d056b6a9, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/2f8b97ce05434d189b0879d1c38c9318, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/58c3117356904f06b3bc4b9a216f4a6e, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/8e6aa46a99164a88abe3527abb210acb, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/c56f1fc0b7764d3989f188593207eb5a, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/db99f21ab99d432fbfdb622a5f8e040f, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/46d2189b128143cbaba9048439b73351, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/9e339863b2b844ba96592008da221246, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/a1617fd9006e4fe79016e8ec8ff15d7d] to archive 2024-11-27T07:26:22,030 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/289954c9202d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T07:26:22,031 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/289954c9202d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/3fafbde684bd455d80f2725242a1eb07 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/3fafbde684bd455d80f2725242a1eb07 2024-11-27T07:26:22,032 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/289954c9202d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/b0a1adabfb6841bdaaf4d2a4581f0040 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/b0a1adabfb6841bdaaf4d2a4581f0040 2024-11-27T07:26:22,032 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/289954c9202d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/498477fe7a9548daa93c462a2f24b842 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/498477fe7a9548daa93c462a2f24b842 2024-11-27T07:26:22,033 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/289954c9202d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/e282058b049a460b83fa3ec126537667 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/e282058b049a460b83fa3ec126537667 2024-11-27T07:26:22,034 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/289954c9202d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/1b65b08c5b0c4d63af85f743429f075a to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/1b65b08c5b0c4d63af85f743429f075a 2024-11-27T07:26:22,034 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/289954c9202d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/6b0dece1caba4c7bb71ae164750bce10 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/6b0dece1caba4c7bb71ae164750bce10 2024-11-27T07:26:22,035 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/289954c9202d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/0e378feb740c4c878a496a4d488e4bc5 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/0e378feb740c4c878a496a4d488e4bc5 2024-11-27T07:26:22,036 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/289954c9202d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/a11c8c23870040b29c0d36e675d3bd6e to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/a11c8c23870040b29c0d36e675d3bd6e 2024-11-27T07:26:22,036 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/289954c9202d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/180862389c0c4704a09325d111a11770 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/180862389c0c4704a09325d111a11770 2024-11-27T07:26:22,037 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/289954c9202d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/38be5e554bcb4481bcf642b9af3e8613 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/38be5e554bcb4481bcf642b9af3e8613 2024-11-27T07:26:22,038 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/289954c9202d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/357dc42b39c048c6a42c3c684d7499db to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/357dc42b39c048c6a42c3c684d7499db 2024-11-27T07:26:22,038 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/289954c9202d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/8f4d1b717d9d46f98d1bde8ec649ffe9 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/8f4d1b717d9d46f98d1bde8ec649ffe9 2024-11-27T07:26:22,039 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/289954c9202d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/681ccb68fa3c4f4fb56b8204d056b6a9 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/681ccb68fa3c4f4fb56b8204d056b6a9 2024-11-27T07:26:22,040 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/289954c9202d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/2f8b97ce05434d189b0879d1c38c9318 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/2f8b97ce05434d189b0879d1c38c9318 2024-11-27T07:26:22,040 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/289954c9202d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/58c3117356904f06b3bc4b9a216f4a6e to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/58c3117356904f06b3bc4b9a216f4a6e 2024-11-27T07:26:22,041 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/289954c9202d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/8e6aa46a99164a88abe3527abb210acb to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/8e6aa46a99164a88abe3527abb210acb 2024-11-27T07:26:22,042 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/289954c9202d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/c56f1fc0b7764d3989f188593207eb5a to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/c56f1fc0b7764d3989f188593207eb5a 2024-11-27T07:26:22,042 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/289954c9202d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/db99f21ab99d432fbfdb622a5f8e040f to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/db99f21ab99d432fbfdb622a5f8e040f 2024-11-27T07:26:22,043 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/289954c9202d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/46d2189b128143cbaba9048439b73351 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/46d2189b128143cbaba9048439b73351 2024-11-27T07:26:22,044 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/289954c9202d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/9e339863b2b844ba96592008da221246 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/9e339863b2b844ba96592008da221246 2024-11-27T07:26:22,044 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/289954c9202d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/a1617fd9006e4fe79016e8ec8ff15d7d to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/a1617fd9006e4fe79016e8ec8ff15d7d 2024-11-27T07:26:22,193 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=322, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/249208ca0dd74e85bb0b1bd86d940b08 2024-11-27T07:26:22,197 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/B/81fbaec571f64acf81260ed859f87a4d is 50, key is test_row_0/B:col10/1732692377519/Put/seqid=0 2024-11-27T07:26:22,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742258_1434 (size=12301) 2024-11-27T07:26:22,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-27T07:26:22,601 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/B/81fbaec571f64acf81260ed859f87a4d 2024-11-27T07:26:22,609 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/C/877a7e0b1ded48afa157e5e1aa5251ab is 50, key is test_row_0/C:col10/1732692377519/Put/seqid=0 2024-11-27T07:26:22,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742259_1435 (size=12301) 2024-11-27T07:26:23,014 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/C/877a7e0b1ded48afa157e5e1aa5251ab 2024-11-27T07:26:23,024 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/A/249208ca0dd74e85bb0b1bd86d940b08 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/249208ca0dd74e85bb0b1bd86d940b08 2024-11-27T07:26:23,027 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/249208ca0dd74e85bb0b1bd86d940b08, entries=150, sequenceid=322, filesize=30.5 K 2024-11-27T07:26:23,028 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/B/81fbaec571f64acf81260ed859f87a4d as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/81fbaec571f64acf81260ed859f87a4d 2024-11-27T07:26:23,030 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/81fbaec571f64acf81260ed859f87a4d, entries=150, sequenceid=322, filesize=12.0 K 2024-11-27T07:26:23,031 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/.tmp/C/877a7e0b1ded48afa157e5e1aa5251ab as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/877a7e0b1ded48afa157e5e1aa5251ab 2024-11-27T07:26:23,034 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/877a7e0b1ded48afa157e5e1aa5251ab, entries=150, sequenceid=322, filesize=12.0 K 2024-11-27T07:26:23,034 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 9cd6e5b712012f5d5b13dc2bcef88d1f in 1658ms, sequenceid=322, compaction requested=true 2024-11-27T07:26:23,038 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/recovered.edits/325.seqid, newMaxSeqId=325, maxSeqId=4 2024-11-27T07:26:23,038 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. 2024-11-27T07:26:23,038 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1676): Region close journal for 9cd6e5b712012f5d5b13dc2bcef88d1f: Waiting for close lock at 1732692381376Running coprocessor pre-close hooks at 1732692381376Disabling compacts and flushes for region at 1732692381376Disabling writes for close at 1732692381376Obtaining lock to block concurrent updates at 1732692381376Preparing flush snapshotting stores in 9cd6e5b712012f5d5b13dc2bcef88d1f at 1732692381376Finished memstore snapshotting TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1732692381377 (+1 ms)Flushing stores of TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f. at 1732692381377Flushing 9cd6e5b712012f5d5b13dc2bcef88d1f/A: creating writer at 1732692381377Flushing mob file 9cd6e5b712012f5d5b13dc2bcef88d1f/A: appending metadata at 1732692381381 (+4 ms)Flushing mob file 9cd6e5b712012f5d5b13dc2bcef88d1f/A: closing flushed file at 1732692381381Flushing 9cd6e5b712012f5d5b13dc2bcef88d1f/A: appending metadata at 1732692381788 (+407 ms)Flushing 9cd6e5b712012f5d5b13dc2bcef88d1f/A: closing flushed file at 1732692381789 (+1 ms)Flushing 9cd6e5b712012f5d5b13dc2bcef88d1f/B: creating writer at 1732692382195 (+406 ms)Flushing 9cd6e5b712012f5d5b13dc2bcef88d1f/B: appending metadata at 1732692382197 (+2 ms)Flushing 9cd6e5b712012f5d5b13dc2bcef88d1f/B: closing flushed file at 1732692382197Flushing 9cd6e5b712012f5d5b13dc2bcef88d1f/C: creating writer at 1732692382605 (+408 ms)Flushing 9cd6e5b712012f5d5b13dc2bcef88d1f/C: appending metadata at 1732692382609 (+4 ms)Flushing 9cd6e5b712012f5d5b13dc2bcef88d1f/C: closing flushed file at 1732692382609Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@cee4c23: reopening flushed file at 1732692383023 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5b86ad5b: reopening flushed file at 1732692383027 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7f64e13e: reopening flushed file at 1732692383031 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 9cd6e5b712012f5d5b13dc2bcef88d1f in 1658ms, sequenceid=322, compaction requested=true at 1732692383034 (+3 ms)Writing region close event to WAL at 1732692383035 (+1 ms)Running coprocessor post-close hooks at 1732692383038 (+3 ms)Closed at 1732692383038 2024-11-27T07:26:23,040 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(157): Closed 9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:26:23,040 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=9cd6e5b712012f5d5b13dc2bcef88d1f, regionState=CLOSED 2024-11-27T07:26:23,041 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=116, ppid=115, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 because future has completed 2024-11-27T07:26:23,042 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-27T07:26:23,043 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; CloseRegionProcedure 9cd6e5b712012f5d5b13dc2bcef88d1f, server=289954c9202d,42117,1732692259671 in 1.8170 sec 2024-11-27T07:26:23,044 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=115, resume processing ppid=114 2024-11-27T07:26:23,044 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, ppid=114, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9cd6e5b712012f5d5b13dc2bcef88d1f, UNASSIGN in 1.8220 sec 2024-11-27T07:26:23,045 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-27T07:26:23,045 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8240 sec 2024-11-27T07:26:23,046 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732692383046"}]},"ts":"1732692383046"} 2024-11-27T07:26:23,047 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-27T07:26:23,047 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-27T07:26:23,049 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8330 sec 2024-11-27T07:26:23,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-27T07:26:23,360 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-27T07:26:23,360 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-27T07:26:23,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T07:26:23,364 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=117, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T07:26:23,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-27T07:26:23,365 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=117, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T07:26:23,368 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:26:23,371 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A, FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B, FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C, FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/recovered.edits] 2024-11-27T07:26:23,374 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/1c78208700f24db6bd058a7dfc100b67 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/1c78208700f24db6bd058a7dfc100b67 2024-11-27T07:26:23,376 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/249208ca0dd74e85bb0b1bd86d940b08 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/249208ca0dd74e85bb0b1bd86d940b08 2024-11-27T07:26:23,377 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/621a8a21d6264e8baa781e262b87d39f to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/A/621a8a21d6264e8baa781e262b87d39f 2024-11-27T07:26:23,379 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/0676e09349b344a1bcf9b7160ee7f8b8 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/0676e09349b344a1bcf9b7160ee7f8b8 2024-11-27T07:26:23,380 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/81fbaec571f64acf81260ed859f87a4d to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/81fbaec571f64acf81260ed859f87a4d 2024-11-27T07:26:23,381 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/961618df80674356b95f431bc60743be to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/B/961618df80674356b95f431bc60743be 2024-11-27T07:26:23,383 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/529225bfc6c0412eb8cca71d18417cba to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/529225bfc6c0412eb8cca71d18417cba 2024-11-27T07:26:23,384 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/55db2297a2494427be1c04b36f59db35 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/55db2297a2494427be1c04b36f59db35 2024-11-27T07:26:23,385 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/877a7e0b1ded48afa157e5e1aa5251ab to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/C/877a7e0b1ded48afa157e5e1aa5251ab 2024-11-27T07:26:23,388 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/recovered.edits/325.seqid to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f/recovered.edits/325.seqid 2024-11-27T07:26:23,388 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:26:23,388 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-27T07:26:23,389 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-27T07:26:23,390 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-27T07:26:23,393 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112708eac5f3d9f84c87a102100fb292c58f_9cd6e5b712012f5d5b13dc2bcef88d1f to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112708eac5f3d9f84c87a102100fb292c58f_9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:26:23,394 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112717c200f61b98477983234c3cfe89163d_9cd6e5b712012f5d5b13dc2bcef88d1f to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112717c200f61b98477983234c3cfe89163d_9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:26:23,395 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411272330fc70083c4441ac8f619aa7419081_9cd6e5b712012f5d5b13dc2bcef88d1f to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411272330fc70083c4441ac8f619aa7419081_9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:26:23,396 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411273cdc2fa0bd29466399a132da6a7dbda8_9cd6e5b712012f5d5b13dc2bcef88d1f to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411273cdc2fa0bd29466399a132da6a7dbda8_9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:26:23,397 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411273e62b0c25fd04adeaf09b03d42c77d03_9cd6e5b712012f5d5b13dc2bcef88d1f to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411273e62b0c25fd04adeaf09b03d42c77d03_9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:26:23,398 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112742a651d45e7e4fdaa3108d66e77c7bd4_9cd6e5b712012f5d5b13dc2bcef88d1f to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112742a651d45e7e4fdaa3108d66e77c7bd4_9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:26:23,399 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112766367639368a4b9c9e887d7b74bafa8b_9cd6e5b712012f5d5b13dc2bcef88d1f to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112766367639368a4b9c9e887d7b74bafa8b_9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:26:23,400 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112767ede66e254e4044ba13cb2a9a220291_9cd6e5b712012f5d5b13dc2bcef88d1f to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112767ede66e254e4044ba13cb2a9a220291_9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:26:23,401 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411277327607443f64e4eb0a51df999d82438_9cd6e5b712012f5d5b13dc2bcef88d1f to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411277327607443f64e4eb0a51df999d82438_9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:26:23,402 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411277ca0f2c46ba843e2b75bd59c4b516aa7_9cd6e5b712012f5d5b13dc2bcef88d1f to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411277ca0f2c46ba843e2b75bd59c4b516aa7_9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:26:23,402 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411279328e4f9584c4754a2b73dc2ed8392ee_9cd6e5b712012f5d5b13dc2bcef88d1f to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411279328e4f9584c4754a2b73dc2ed8392ee_9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:26:23,403 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127a7128ca2f9a2463088c2636694a631d7_9cd6e5b712012f5d5b13dc2bcef88d1f to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127a7128ca2f9a2463088c2636694a631d7_9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:26:23,404 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127d02469c8ffd145b9b75983d29ed4d9e3_9cd6e5b712012f5d5b13dc2bcef88d1f to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127d02469c8ffd145b9b75983d29ed4d9e3_9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:26:23,405 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127e1b60cee38634529bf6c4f6a8da5a47a_9cd6e5b712012f5d5b13dc2bcef88d1f to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127e1b60cee38634529bf6c4f6a8da5a47a_9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:26:23,405 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127ea91402c329d4632838f965788a274ef_9cd6e5b712012f5d5b13dc2bcef88d1f to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127ea91402c329d4632838f965788a274ef_9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:26:23,406 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127f4e94483e596483c8950f368812fc5b8_9cd6e5b712012f5d5b13dc2bcef88d1f to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127f4e94483e596483c8950f368812fc5b8_9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:26:23,407 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127fe81ab30692c4e499a1525861a1c9159_9cd6e5b712012f5d5b13dc2bcef88d1f to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127fe81ab30692c4e499a1525861a1c9159_9cd6e5b712012f5d5b13dc2bcef88d1f 2024-11-27T07:26:23,407 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-27T07:26:23,409 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=117, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T07:26:23,410 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-27T07:26:23,412 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-27T07:26:23,413 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=117, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T07:26:23,413 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-27T07:26:23,413 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732692383413"}]},"ts":"9223372036854775807"} 2024-11-27T07:26:23,415 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-27T07:26:23,415 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 9cd6e5b712012f5d5b13dc2bcef88d1f, NAME => 'TestAcidGuarantees,,1732692353648.9cd6e5b712012f5d5b13dc2bcef88d1f.', STARTKEY => '', ENDKEY => ''}] 2024-11-27T07:26:23,415 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-27T07:26:23,415 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732692383415"}]},"ts":"9223372036854775807"} 2024-11-27T07:26:23,417 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-27T07:26:23,417 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=117, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T07:26:23,418 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 57 msec 2024-11-27T07:26:23,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-27T07:26:23,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-27T07:26:23,470 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-27T07:26:23,484 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=239 (was 240), OpenFileDescriptor=449 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=506 (was 481) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4567 (was 4457) - AvailableMemoryMB LEAK? - 2024-11-27T07:26:23,493 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=239, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=506, ProcessCount=11, AvailableMemoryMB=4567 2024-11-27T07:26:23,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-27T07:26:23,495 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T07:26:23,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] procedure2.ProcedureExecutor(1139): Stored pid=118, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-27T07:26:23,497 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T07:26:23,497 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:26:23,497 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 118 2024-11-27T07:26:23,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-27T07:26:23,498 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T07:26:23,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742260_1436 (size=959) 2024-11-27T07:26:23,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-27T07:26:23,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-27T07:26:23,909 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4 2024-11-27T07:26:23,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742261_1437 (size=53) 2024-11-27T07:26:24,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-27T07:26:24,319 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T07:26:24,320 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 8dcfd8a4aaa1fef77f3b9d89e832c182, disabling compactions & flushes 2024-11-27T07:26:24,320 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:24,320 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:24,320 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. after waiting 0 ms 2024-11-27T07:26:24,320 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:24,320 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:24,320 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8dcfd8a4aaa1fef77f3b9d89e832c182: Waiting for close lock at 1732692384320Disabling compacts and flushes for region at 1732692384320Disabling writes for close at 1732692384320Writing region close event to WAL at 1732692384320Closed at 1732692384320 2024-11-27T07:26:24,323 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T07:26:24,323 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732692384323"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732692384323"}]},"ts":"1732692384323"} 2024-11-27T07:26:24,327 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T07:26:24,328 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T07:26:24,328 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732692384328"}]},"ts":"1732692384328"} 2024-11-27T07:26:24,329 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-27T07:26:24,330 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8dcfd8a4aaa1fef77f3b9d89e832c182, ASSIGN}] 2024-11-27T07:26:24,331 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8dcfd8a4aaa1fef77f3b9d89e832c182, ASSIGN 2024-11-27T07:26:24,332 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8dcfd8a4aaa1fef77f3b9d89e832c182, ASSIGN; state=OFFLINE, location=289954c9202d,42117,1732692259671; forceNewPlan=false, retain=false 2024-11-27T07:26:24,483 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=8dcfd8a4aaa1fef77f3b9d89e832c182, regionState=OPENING, regionLocation=289954c9202d,42117,1732692259671 2024-11-27T07:26:24,486 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8dcfd8a4aaa1fef77f3b9d89e832c182, ASSIGN because future has completed 2024-11-27T07:26:24,486 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671}] 2024-11-27T07:26:24,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-27T07:26:24,642 INFO [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:24,642 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7752): Opening region: {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} 2024-11-27T07:26:24,643 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 8dcfd8a4aaa1fef77f3b9d89e832c182 2024-11-27T07:26:24,643 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T07:26:24,643 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7794): checking encryption for 8dcfd8a4aaa1fef77f3b9d89e832c182 2024-11-27T07:26:24,643 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7797): checking classloading for 8dcfd8a4aaa1fef77f3b9d89e832c182 2024-11-27T07:26:24,645 INFO [StoreOpener-8dcfd8a4aaa1fef77f3b9d89e832c182-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 8dcfd8a4aaa1fef77f3b9d89e832c182 2024-11-27T07:26:24,646 INFO [StoreOpener-8dcfd8a4aaa1fef77f3b9d89e832c182-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8dcfd8a4aaa1fef77f3b9d89e832c182 columnFamilyName A 2024-11-27T07:26:24,646 DEBUG [StoreOpener-8dcfd8a4aaa1fef77f3b9d89e832c182-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:26:24,647 INFO [StoreOpener-8dcfd8a4aaa1fef77f3b9d89e832c182-1 {}] regionserver.HStore(327): Store=8dcfd8a4aaa1fef77f3b9d89e832c182/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T07:26:24,647 INFO [StoreOpener-8dcfd8a4aaa1fef77f3b9d89e832c182-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 8dcfd8a4aaa1fef77f3b9d89e832c182 2024-11-27T07:26:24,648 INFO [StoreOpener-8dcfd8a4aaa1fef77f3b9d89e832c182-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8dcfd8a4aaa1fef77f3b9d89e832c182 columnFamilyName B 2024-11-27T07:26:24,648 DEBUG [StoreOpener-8dcfd8a4aaa1fef77f3b9d89e832c182-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:26:24,648 INFO [StoreOpener-8dcfd8a4aaa1fef77f3b9d89e832c182-1 {}] regionserver.HStore(327): Store=8dcfd8a4aaa1fef77f3b9d89e832c182/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T07:26:24,648 INFO [StoreOpener-8dcfd8a4aaa1fef77f3b9d89e832c182-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 8dcfd8a4aaa1fef77f3b9d89e832c182 2024-11-27T07:26:24,649 INFO [StoreOpener-8dcfd8a4aaa1fef77f3b9d89e832c182-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8dcfd8a4aaa1fef77f3b9d89e832c182 columnFamilyName C 2024-11-27T07:26:24,649 DEBUG [StoreOpener-8dcfd8a4aaa1fef77f3b9d89e832c182-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:26:24,649 INFO [StoreOpener-8dcfd8a4aaa1fef77f3b9d89e832c182-1 {}] regionserver.HStore(327): Store=8dcfd8a4aaa1fef77f3b9d89e832c182/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T07:26:24,650 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1038): replaying wal for 8dcfd8a4aaa1fef77f3b9d89e832c182 2024-11-27T07:26:24,650 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182 2024-11-27T07:26:24,651 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182 2024-11-27T07:26:24,651 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1048): stopping wal replay for 8dcfd8a4aaa1fef77f3b9d89e832c182 2024-11-27T07:26:24,652 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1060): Cleaning up temporary data for 8dcfd8a4aaa1fef77f3b9d89e832c182 2024-11-27T07:26:24,652 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T07:26:24,653 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1093): writing seq id for 8dcfd8a4aaa1fef77f3b9d89e832c182 2024-11-27T07:26:24,655 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T07:26:24,656 INFO [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1114): Opened 8dcfd8a4aaa1fef77f3b9d89e832c182; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66437517, jitterRate=-0.010003849864006042}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T07:26:24,656 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8dcfd8a4aaa1fef77f3b9d89e832c182 2024-11-27T07:26:24,656 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1006): Region open journal for 8dcfd8a4aaa1fef77f3b9d89e832c182: Running coprocessor pre-open hook at 1732692384644Writing region info on filesystem at 1732692384644Initializing all the Stores at 1732692384644Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732692384644Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732692384645 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732692384645Cleaning up temporary data from old regions at 1732692384652 (+7 ms)Running coprocessor post-open hooks at 1732692384656 (+4 ms)Region opened successfully at 1732692384656 2024-11-27T07:26:24,657 INFO [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., pid=120, masterSystemTime=1732692384640 2024-11-27T07:26:24,659 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:24,659 INFO [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:24,659 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=8dcfd8a4aaa1fef77f3b9d89e832c182, regionState=OPEN, openSeqNum=2, regionLocation=289954c9202d,42117,1732692259671 2024-11-27T07:26:24,660 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=120, ppid=119, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 because future has completed 2024-11-27T07:26:24,662 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-27T07:26:24,663 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; OpenRegionProcedure 8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 in 175 msec 2024-11-27T07:26:24,664 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=119, resume processing ppid=118 2024-11-27T07:26:24,664 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, ppid=118, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8dcfd8a4aaa1fef77f3b9d89e832c182, ASSIGN in 333 msec 2024-11-27T07:26:24,665 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T07:26:24,665 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732692384665"}]},"ts":"1732692384665"} 2024-11-27T07:26:24,666 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-27T07:26:24,667 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T07:26:24,668 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1720 sec 2024-11-27T07:26:25,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-27T07:26:25,649 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-27T07:26:25,650 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b8abc8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:26:25,650 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 289954c9202d,43057,-1 for getting cluster id 2024-11-27T07:26:25,651 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T07:26:25,652 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '09f43d3f-c175-4713-a9ce-038f5695b74b' 2024-11-27T07:26:25,652 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T07:26:25,652 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "09f43d3f-c175-4713-a9ce-038f5695b74b" 2024-11-27T07:26:25,653 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d9095a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:26:25,653 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [289954c9202d,43057,-1] 2024-11-27T07:26:25,653 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T07:26:25,653 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:26:25,655 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51688, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T07:26:25,656 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56d2e1d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:26:25,656 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T07:26:25,658 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=289954c9202d,42117,1732692259671, seqNum=-1] 2024-11-27T07:26:25,658 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T07:26:25,660 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34392, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T07:26:25,661 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-27T07:26:25,662 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 289954c9202d,43057,1732692258941 2024-11-27T07:26:25,662 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@145d06a7 2024-11-27T07:26:25,663 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-27T07:26:25,664 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51702, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-27T07:26:25,665 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26bd122f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:26:25,665 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 289954c9202d,43057,-1 for getting cluster id 2024-11-27T07:26:25,665 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T07:26:25,666 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '09f43d3f-c175-4713-a9ce-038f5695b74b' 2024-11-27T07:26:25,666 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T07:26:25,666 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "09f43d3f-c175-4713-a9ce-038f5695b74b" 2024-11-27T07:26:25,666 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68cbdbfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:26:25,666 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [289954c9202d,43057,-1] 2024-11-27T07:26:25,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T07:26:25,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:26:25,667 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51732, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T07:26:25,668 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@743c0781, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:26:25,669 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f0c05eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:26:25,669 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 289954c9202d,43057,-1 for getting cluster id 2024-11-27T07:26:25,669 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T07:26:25,669 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '09f43d3f-c175-4713-a9ce-038f5695b74b' 2024-11-27T07:26:25,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T07:26:25,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "09f43d3f-c175-4713-a9ce-038f5695b74b" 2024-11-27T07:26:25,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e6783d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:26:25,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [289954c9202d,43057,-1] 2024-11-27T07:26:25,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T07:26:25,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:26:25,671 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51760, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T07:26:25,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2898d206, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:26:25,672 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79d7458, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:26:25,672 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 289954c9202d,43057,-1 for getting cluster id 2024-11-27T07:26:25,672 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T07:26:25,673 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '09f43d3f-c175-4713-a9ce-038f5695b74b' 2024-11-27T07:26:25,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T07:26:25,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "09f43d3f-c175-4713-a9ce-038f5695b74b" 2024-11-27T07:26:25,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7669d351, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:26:25,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [289954c9202d,43057,-1] 2024-11-27T07:26:25,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T07:26:25,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:26:25,674 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51774, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T07:26:25,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4eb92597, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:26:25,675 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32c61cf9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:26:25,675 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 289954c9202d,43057,-1 for getting cluster id 2024-11-27T07:26:25,675 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T07:26:25,676 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '09f43d3f-c175-4713-a9ce-038f5695b74b' 2024-11-27T07:26:25,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T07:26:25,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "09f43d3f-c175-4713-a9ce-038f5695b74b" 2024-11-27T07:26:25,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c82d02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:26:25,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [289954c9202d,43057,-1] 2024-11-27T07:26:25,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T07:26:25,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:26:25,677 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51790, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T07:26:25,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bd1b757, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:26:25,678 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c296bf5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:26:25,678 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 289954c9202d,43057,-1 for getting cluster id 2024-11-27T07:26:25,678 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T07:26:25,678 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '09f43d3f-c175-4713-a9ce-038f5695b74b' 2024-11-27T07:26:25,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T07:26:25,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "09f43d3f-c175-4713-a9ce-038f5695b74b" 2024-11-27T07:26:25,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bab9d24, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:26:25,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [289954c9202d,43057,-1] 2024-11-27T07:26:25,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T07:26:25,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:26:25,680 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51798, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T07:26:25,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41eb5502, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:26:25,680 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ff7674f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:26:25,681 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 289954c9202d,43057,-1 for getting cluster id 2024-11-27T07:26:25,681 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T07:26:25,681 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '09f43d3f-c175-4713-a9ce-038f5695b74b' 2024-11-27T07:26:25,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T07:26:25,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "09f43d3f-c175-4713-a9ce-038f5695b74b" 2024-11-27T07:26:25,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7abb3b72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:26:25,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [289954c9202d,43057,-1] 2024-11-27T07:26:25,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T07:26:25,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:26:25,682 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51826, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T07:26:25,683 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73e77522, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:26:25,683 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@672fe15e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:26:25,683 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 289954c9202d,43057,-1 for getting cluster id 2024-11-27T07:26:25,683 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T07:26:25,684 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '09f43d3f-c175-4713-a9ce-038f5695b74b' 2024-11-27T07:26:25,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T07:26:25,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "09f43d3f-c175-4713-a9ce-038f5695b74b" 2024-11-27T07:26:25,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d79e606, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:26:25,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [289954c9202d,43057,-1] 2024-11-27T07:26:25,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T07:26:25,685 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:26:25,685 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51846, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T07:26:25,686 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@315cd1e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:26:25,686 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71238b4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:26:25,686 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 289954c9202d,43057,-1 for getting cluster id 2024-11-27T07:26:25,686 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T07:26:25,687 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '09f43d3f-c175-4713-a9ce-038f5695b74b' 2024-11-27T07:26:25,687 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T07:26:25,687 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "09f43d3f-c175-4713-a9ce-038f5695b74b" 2024-11-27T07:26:25,687 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56693319, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:26:25,687 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [289954c9202d,43057,-1] 2024-11-27T07:26:25,687 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T07:26:25,688 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:26:25,688 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51862, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T07:26:25,689 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b3cefd7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:26:25,689 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@625b319d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:26:25,689 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 289954c9202d,43057,-1 for getting cluster id 2024-11-27T07:26:25,689 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T07:26:25,690 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '09f43d3f-c175-4713-a9ce-038f5695b74b' 2024-11-27T07:26:25,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T07:26:25,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "09f43d3f-c175-4713-a9ce-038f5695b74b" 2024-11-27T07:26:25,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b6fc262, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:26:25,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [289954c9202d,43057,-1] 2024-11-27T07:26:25,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T07:26:25,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:26:25,691 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51880, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T07:26:25,692 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64d41b53, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:26:25,693 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49cd7b43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:26:25,693 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 289954c9202d,43057,-1 for getting cluster id 2024-11-27T07:26:25,693 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T07:26:25,693 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '09f43d3f-c175-4713-a9ce-038f5695b74b' 2024-11-27T07:26:25,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T07:26:25,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "09f43d3f-c175-4713-a9ce-038f5695b74b" 2024-11-27T07:26:25,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@387e17d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:26:25,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [289954c9202d,43057,-1] 2024-11-27T07:26:25,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T07:26:25,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:26:25,695 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51888, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T07:26:25,695 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f12952e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:26:25,696 DEBUG [Thread-1966 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T07:26:25,696 DEBUG [Thread-1975 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T07:26:25,696 DEBUG [Thread-1981 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T07:26:25,696 DEBUG [Thread-1970 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T07:26:25,696 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T07:26:25,696 DEBUG [Thread-1977 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T07:26:25,697 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=289954c9202d,42117,1732692259671, seqNum=-1] 2024-11-27T07:26:25,697 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=289954c9202d,42117,1732692259671, seqNum=-1] 2024-11-27T07:26:25,697 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=289954c9202d,42117,1732692259671, seqNum=-1] 2024-11-27T07:26:25,697 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T07:26:25,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=289954c9202d,42117,1732692259671, seqNum=-1] 2024-11-27T07:26:25,697 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=289954c9202d,42117,1732692259671, seqNum=-1] 2024-11-27T07:26:25,697 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T07:26:25,698 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T07:26:25,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T07:26:25,698 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T07:26:25,698 DEBUG [Thread-1968 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T07:26:25,699 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34398, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T07:26:25,699 DEBUG [Thread-1964 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T07:26:25,699 DEBUG [Thread-1983 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T07:26:25,699 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34408, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T07:26:25,699 DEBUG [Thread-1972 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T07:26:25,699 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34404, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T07:26:25,699 DEBUG [Thread-1979 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T07:26:25,699 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34416, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T07:26:25,700 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34430, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T07:26:25,700 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=289954c9202d,42117,1732692259671, seqNum=-1] 2024-11-27T07:26:25,700 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T07:26:25,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43057 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-27T07:26:25,700 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=289954c9202d,42117,1732692259671, seqNum=-1] 2024-11-27T07:26:25,701 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T07:26:25,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=289954c9202d,42117,1732692259671, seqNum=-1] 2024-11-27T07:26:25,701 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=289954c9202d,42117,1732692259671, seqNum=-1] 2024-11-27T07:26:25,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T07:26:25,702 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T07:26:25,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2] 2024-11-27T07:26:25,702 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=289954c9202d,42117,1732692259671, seqNum=-1] 2024-11-27T07:26:25,702 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T07:26:25,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-27T07:26:25,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2] 2024-11-27T07:26:25,703 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34446, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T07:26:25,703 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34440, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T07:26:25,703 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34444, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T07:26:25,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2] 2024-11-27T07:26:25,703 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34450, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T07:26:25,703 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34454, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T07:26:25,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2] 2024-11-27T07:26:25,704 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2] 2024-11-27T07:26:25,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2] 2024-11-27T07:26:25,705 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2] 2024-11-27T07:26:25,705 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2] 2024-11-27T07:26:25,705 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T07:26:25,706 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2] 2024-11-27T07:26:25,706 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2] 2024-11-27T07:26:25,706 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T07:26:25,706 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T07:26:25,712 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T07:26:25,712 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8dcfd8a4aaa1fef77f3b9d89e832c182 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-27T07:26:25,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 8dcfd8a4aaa1fef77f3b9d89e832c182 2024-11-27T07:26:25,732 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/21462d73ce4c42bd985073e454240f3f is 50, key is test_row_0/A:col10/1732692385710/Put/seqid=0 2024-11-27T07:26:25,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742262_1438 (size=14341) 2024-11-27T07:26:25,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:25,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:25,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34408 deadline: 1732692445760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:25,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34446 deadline: 1732692445761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:25,771 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:25,771 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:25,771 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:25,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:25,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:25,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:25,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:25,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34398 deadline: 1732692445769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:25,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:25,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:25,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:25,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:25,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34440 deadline: 1732692445771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:25,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:25,779 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:25,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34450 deadline: 1732692445771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:25,779 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:25,779 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:25,779 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:25,779 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:25,779 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:25,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-27T07:26:25,859 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-27T07:26:25,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:25,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. as already flushing 2024-11-27T07:26:25,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:25,859 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:25,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:25,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:25,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:25,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:25,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34446 deadline: 1732692445878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:25,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34408 deadline: 1732692445879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:25,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:25,885 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:25,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34398 deadline: 1732692445879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:25,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:25,885 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:25,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:25,885 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:25,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:25,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:25,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:25,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:25,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:25,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34440 deadline: 1732692445889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:25,896 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:25,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:25,896 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:25,896 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:25,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34450 deadline: 1732692445889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:25,896 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:25,896 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:25,896 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:26,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-27T07:26:26,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:26,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. as already flushing 2024-11-27T07:26:26,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:26,012 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:26,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:26,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:26,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-27T07:26:26,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:26,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34446 deadline: 1732692446089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:26,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:26,092 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:26,092 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:26,092 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:26,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34408 deadline: 1732692446089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:26,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:26,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:26,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:26,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:26,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34398 deadline: 1732692446089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:26,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:26,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:26,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:26,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:26,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34440 deadline: 1732692446099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:26,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:26,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:26,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34450 deadline: 1732692446099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:26,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:26,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:26,104 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:26,104 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:26,104 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:26,136 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/21462d73ce4c42bd985073e454240f3f 2024-11-27T07:26:26,157 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/88beedf7496040838601d8ce45671f31 is 50, key is test_row_0/B:col10/1732692385710/Put/seqid=0 2024-11-27T07:26:26,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742263_1439 (size=12001) 2024-11-27T07:26:26,161 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/88beedf7496040838601d8ce45671f31 2024-11-27T07:26:26,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-27T07:26:26,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:26,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. as already flushing 2024-11-27T07:26:26,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:26,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:26,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:26,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:26,185 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/d79c221309a94958a6d5207638ae8653 is 50, key is test_row_0/C:col10/1732692385710/Put/seqid=0 2024-11-27T07:26:26,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742264_1440 (size=12001) 2024-11-27T07:26:26,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-27T07:26:26,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:26,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. as already flushing 2024-11-27T07:26:26,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:26,318 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:26,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:26,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:26,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-27T07:26:26,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:26,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34446 deadline: 1732692446399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:26,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:26,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34408 deadline: 1732692446399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:26,400 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:26,400 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:26,400 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:26,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:26,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:26,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34398 deadline: 1732692446399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:26,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:26,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:26,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:26,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:26,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:26,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:26,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34440 deadline: 1732692446409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:26,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:26,413 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:26,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34450 deadline: 1732692446409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:26,413 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:26,413 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:26,413 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:26,413 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:26,413 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:26,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-27T07:26:26,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:26,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. as already flushing 2024-11-27T07:26:26,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:26,470 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:26,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:26,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:26,588 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/d79c221309a94958a6d5207638ae8653 2024-11-27T07:26:26,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/21462d73ce4c42bd985073e454240f3f as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/21462d73ce4c42bd985073e454240f3f 2024-11-27T07:26:26,597 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/21462d73ce4c42bd985073e454240f3f, entries=200, sequenceid=14, filesize=14.0 K 2024-11-27T07:26:26,597 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/88beedf7496040838601d8ce45671f31 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/88beedf7496040838601d8ce45671f31 2024-11-27T07:26:26,600 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/88beedf7496040838601d8ce45671f31, entries=150, sequenceid=14, filesize=11.7 K 2024-11-27T07:26:26,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/d79c221309a94958a6d5207638ae8653 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/d79c221309a94958a6d5207638ae8653 2024-11-27T07:26:26,604 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/d79c221309a94958a6d5207638ae8653, entries=150, sequenceid=14, filesize=11.7 K 2024-11-27T07:26:26,604 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 8dcfd8a4aaa1fef77f3b9d89e832c182 in 892ms, sequenceid=14, compaction requested=false 2024-11-27T07:26:26,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8dcfd8a4aaa1fef77f3b9d89e832c182: 2024-11-27T07:26:26,630 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-27T07:26:26,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:26,630 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 8dcfd8a4aaa1fef77f3b9d89e832c182 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-27T07:26:26,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/d4392dd3e2694f0e9e7faebcf9a12278 is 50, key is test_row_0/A:col10/1732692385755/Put/seqid=0 2024-11-27T07:26:26,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742265_1441 (size=12001) 2024-11-27T07:26:26,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-27T07:26:26,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 8dcfd8a4aaa1fef77f3b9d89e832c182 2024-11-27T07:26:26,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. as already flushing 2024-11-27T07:26:26,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:26,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34398 deadline: 1732692446924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:26,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:26,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:26,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:26,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:26,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34440 deadline: 1732692446926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:26,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:26,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:26,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34450 deadline: 1732692446927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:26,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:26,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:26,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:26,935 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:26,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34446 deadline: 1732692446927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:26,935 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:26,935 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:26,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:26,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34408 deadline: 1732692446931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:26,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:26,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:26,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:26,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:26,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:26,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:27,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:27,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34398 deadline: 1732692447038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:27,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:27,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:27,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34440 deadline: 1732692447039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:27,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:27,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:27,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:27,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34450 deadline: 1732692447039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:27,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:27,042 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/d4392dd3e2694f0e9e7faebcf9a12278 2024-11-27T07:26:27,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:27,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:27,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:27,042 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:27,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34446 deadline: 1732692447039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:27,042 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:27,042 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:27,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:27,042 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:27,042 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:27,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34408 deadline: 1732692447039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:27,042 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:27,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:27,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:27,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:27,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/4719df548a2d4ea9a8c3e257db5a8672 is 50, key is test_row_0/B:col10/1732692385755/Put/seqid=0 2024-11-27T07:26:27,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742266_1442 (size=12001) 2024-11-27T07:26:27,068 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/4719df548a2d4ea9a8c3e257db5a8672 2024-11-27T07:26:27,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/d48748f08cb4494eaeba821c76ded266 is 50, key is test_row_0/C:col10/1732692385755/Put/seqid=0 2024-11-27T07:26:27,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742267_1443 (size=12001) 2024-11-27T07:26:27,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:27,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34398 deadline: 1732692447249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:27,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:27,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34440 deadline: 1732692447249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:27,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:27,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:27,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:27,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:27,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:27,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:27,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:27,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34450 deadline: 1732692447249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:27,255 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:27,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:27,255 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:27,255 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:27,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34446 deadline: 1732692447249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:27,256 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:27,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:27,256 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:27,256 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:27,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34408 deadline: 1732692447249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:27,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:27,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:27,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:27,482 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/d48748f08cb4494eaeba821c76ded266 2024-11-27T07:26:27,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/d4392dd3e2694f0e9e7faebcf9a12278 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/d4392dd3e2694f0e9e7faebcf9a12278 2024-11-27T07:26:27,492 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/d4392dd3e2694f0e9e7faebcf9a12278, entries=150, sequenceid=37, filesize=11.7 K 2024-11-27T07:26:27,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/4719df548a2d4ea9a8c3e257db5a8672 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/4719df548a2d4ea9a8c3e257db5a8672 2024-11-27T07:26:27,498 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/4719df548a2d4ea9a8c3e257db5a8672, entries=150, sequenceid=37, filesize=11.7 K 2024-11-27T07:26:27,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/d48748f08cb4494eaeba821c76ded266 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/d48748f08cb4494eaeba821c76ded266 2024-11-27T07:26:27,507 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/d48748f08cb4494eaeba821c76ded266, entries=150, sequenceid=37, filesize=11.7 K 2024-11-27T07:26:27,508 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 8dcfd8a4aaa1fef77f3b9d89e832c182 in 878ms, sequenceid=37, compaction requested=false 2024-11-27T07:26:27,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 8dcfd8a4aaa1fef77f3b9d89e832c182: 2024-11-27T07:26:27,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:27,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-27T07:26:27,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-27T07:26:27,514 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-27T07:26:27,514 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8040 sec 2024-11-27T07:26:27,517 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.8190 sec 2024-11-27T07:26:27,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 8dcfd8a4aaa1fef77f3b9d89e832c182 2024-11-27T07:26:27,565 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T07:26:27,565 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8dcfd8a4aaa1fef77f3b9d89e832c182 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-27T07:26:27,569 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/a469ef39b2164542aad8b5f8fea486c0 is 50, key is test_row_0/A:col10/1732692387563/Put/seqid=0 2024-11-27T07:26:27,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742268_1444 (size=14341) 2024-11-27T07:26:27,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:27,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34440 deadline: 1732692447621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:27,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:27,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:27,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:27,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:27,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34408 deadline: 1732692447622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:27,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:27,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:27,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:27,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:27,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34450 deadline: 1732692447624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:27,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:27,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34446 deadline: 1732692447625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:27,639 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:27,639 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:27,639 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:27,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:27,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34398 deadline: 1732692447636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:27,639 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:27,639 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:27,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:27,639 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:27,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:27,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:27,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:27,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34440 deadline: 1732692447739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:27,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:27,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:27,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:27,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:27,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34408 deadline: 1732692447749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:27,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:27,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:27,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:27,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:27,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34450 deadline: 1732692447749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:27,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:27,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34446 deadline: 1732692447749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:27,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:27,756 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:27,755 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:27,756 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:27,756 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:27,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34398 deadline: 1732692447749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:27,756 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:27,756 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:27,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:27,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:27,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:27,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-27T07:26:27,849 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T07:26:27,850 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T07:26:27,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-27T07:26:27,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-27T07:26:27,851 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T07:26:27,852 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T07:26:27,852 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T07:26:27,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:27,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34440 deadline: 1732692447949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:27,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:27,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:27,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:27,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-27T07:26:27,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:27,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34408 deadline: 1732692447959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:27,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:27,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:27,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:27,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:27,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34450 deadline: 1732692447959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:27,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:27,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:27,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:27,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:27,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34446 deadline: 1732692447960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:27,968 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:27,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:27,968 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:27,968 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:27,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34398 deadline: 1732692447960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:27,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:27,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:27,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:28,004 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-27T07:26:28,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:28,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. as already flushing 2024-11-27T07:26:28,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:28,004 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:28,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:28,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:28,009 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/a469ef39b2164542aad8b5f8fea486c0 2024-11-27T07:26:28,015 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/03d3c65033a14472a4a36797b39af996 is 50, key is test_row_0/B:col10/1732692387563/Put/seqid=0 2024-11-27T07:26:28,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742269_1445 (size=12001) 2024-11-27T07:26:28,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-27T07:26:28,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:28,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. as already flushing 2024-11-27T07:26:28,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:28,157 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:28,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:28,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:28,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-27T07:26:28,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:28,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34440 deadline: 1732692448259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:28,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:28,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:28,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:28,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:28,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34408 deadline: 1732692448279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:28,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:28,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:28,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:28,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34450 deadline: 1732692448279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:28,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:28,285 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:28,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:28,285 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:28,285 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:28,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34446 deadline: 1732692448279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:28,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:28,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34398 deadline: 1732692448280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:28,285 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:28,285 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:28,285 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:28,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:28,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:28,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:28,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-27T07:26:28,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:28,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. as already flushing 2024-11-27T07:26:28,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:28,310 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:28,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:28,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:28,418 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/03d3c65033a14472a4a36797b39af996 2024-11-27T07:26:28,427 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/4bca193f361e4334a830f52cb235f638 is 50, key is test_row_0/C:col10/1732692387563/Put/seqid=0 2024-11-27T07:26:28,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742270_1446 (size=12001) 2024-11-27T07:26:28,452 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/4bca193f361e4334a830f52cb235f638 2024-11-27T07:26:28,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/a469ef39b2164542aad8b5f8fea486c0 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/a469ef39b2164542aad8b5f8fea486c0 2024-11-27T07:26:28,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-27T07:26:28,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:28,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. as already flushing 2024-11-27T07:26:28,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:28,463 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:28,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:28,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:28,470 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/a469ef39b2164542aad8b5f8fea486c0, entries=200, sequenceid=51, filesize=14.0 K 2024-11-27T07:26:28,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/03d3c65033a14472a4a36797b39af996 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/03d3c65033a14472a4a36797b39af996 2024-11-27T07:26:28,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-27T07:26:28,480 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/03d3c65033a14472a4a36797b39af996, entries=150, sequenceid=51, filesize=11.7 K 2024-11-27T07:26:28,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/4bca193f361e4334a830f52cb235f638 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/4bca193f361e4334a830f52cb235f638 2024-11-27T07:26:28,491 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/4bca193f361e4334a830f52cb235f638, entries=150, sequenceid=51, filesize=11.7 K 2024-11-27T07:26:28,492 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 8dcfd8a4aaa1fef77f3b9d89e832c182 in 927ms, sequenceid=51, compaction requested=true 2024-11-27T07:26:28,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8dcfd8a4aaa1fef77f3b9d89e832c182: 2024-11-27T07:26:28,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8dcfd8a4aaa1fef77f3b9d89e832c182:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T07:26:28,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:26:28,492 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:26:28,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8dcfd8a4aaa1fef77f3b9d89e832c182:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T07:26:28,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:26:28,492 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:26:28,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8dcfd8a4aaa1fef77f3b9d89e832c182:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T07:26:28,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:26:28,493 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:26:28,493 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1541): 8dcfd8a4aaa1fef77f3b9d89e832c182/A is initiating minor compaction (all files) 2024-11-27T07:26:28,493 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8dcfd8a4aaa1fef77f3b9d89e832c182/A in TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:28,493 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/21462d73ce4c42bd985073e454240f3f, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/d4392dd3e2694f0e9e7faebcf9a12278, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/a469ef39b2164542aad8b5f8fea486c0] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp, totalSize=39.7 K 2024-11-27T07:26:28,494 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:26:28,494 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21462d73ce4c42bd985073e454240f3f, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732692385706 2024-11-27T07:26:28,494 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 8dcfd8a4aaa1fef77f3b9d89e832c182/B is initiating minor compaction (all files) 2024-11-27T07:26:28,494 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8dcfd8a4aaa1fef77f3b9d89e832c182/B in TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:28,494 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/88beedf7496040838601d8ce45671f31, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/4719df548a2d4ea9a8c3e257db5a8672, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/03d3c65033a14472a4a36797b39af996] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp, totalSize=35.2 K 2024-11-27T07:26:28,494 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4392dd3e2694f0e9e7faebcf9a12278, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732692385755 2024-11-27T07:26:28,494 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 88beedf7496040838601d8ce45671f31, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732692385706 2024-11-27T07:26:28,494 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting a469ef39b2164542aad8b5f8fea486c0, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732692386923 2024-11-27T07:26:28,494 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 4719df548a2d4ea9a8c3e257db5a8672, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732692385755 2024-11-27T07:26:28,495 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 03d3c65033a14472a4a36797b39af996, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732692386926 2024-11-27T07:26:28,513 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8dcfd8a4aaa1fef77f3b9d89e832c182#A#compaction#369 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:26:28,513 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8dcfd8a4aaa1fef77f3b9d89e832c182#B#compaction#370 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:26:28,513 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/699e81b22f574df590cbebfb0fe86886 is 50, key is test_row_0/A:col10/1732692387563/Put/seqid=0 2024-11-27T07:26:28,513 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/f843736d5a9a41faa471db9eb4eea69d is 50, key is test_row_0/B:col10/1732692387563/Put/seqid=0 2024-11-27T07:26:28,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742272_1448 (size=12104) 2024-11-27T07:26:28,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742271_1447 (size=12104) 2024-11-27T07:26:28,563 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T07:26:28,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-27T07:26:28,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:28,618 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 8dcfd8a4aaa1fef77f3b9d89e832c182 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-27T07:26:28,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/2fc98998b96547bbbf3a33964abf3496 is 50, key is test_row_0/A:col10/1732692387634/Put/seqid=0 2024-11-27T07:26:28,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742273_1449 (size=12001) 2024-11-27T07:26:28,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 8dcfd8a4aaa1fef77f3b9d89e832c182 2024-11-27T07:26:28,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. as already flushing 2024-11-27T07:26:28,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:28,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34450 deadline: 1732692448799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:28,805 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:28,806 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:28,806 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:28,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:28,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34408 deadline: 1732692448799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:28,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:28,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:28,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:28,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:28,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34446 deadline: 1732692448804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:28,809 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:28,809 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:28,809 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:28,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:28,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34398 deadline: 1732692448804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:28,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:28,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:28,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:28,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34440 deadline: 1732692448805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:28,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:28,814 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:28,814 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:28,814 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:28,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:28,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34450 deadline: 1732692448909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:28,916 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:28,917 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:28,917 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:28,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:28,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34446 deadline: 1732692448919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:28,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:28,927 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:28,927 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:28,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34398 deadline: 1732692448919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:28,927 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:28,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:28,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:28,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:28,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:28,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34440 deadline: 1732692448919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:28,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:28,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:28,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:28,933 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/699e81b22f574df590cbebfb0fe86886 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/699e81b22f574df590cbebfb0fe86886 2024-11-27T07:26:28,933 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/f843736d5a9a41faa471db9eb4eea69d as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/f843736d5a9a41faa471db9eb4eea69d 2024-11-27T07:26:28,939 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8dcfd8a4aaa1fef77f3b9d89e832c182/B of 8dcfd8a4aaa1fef77f3b9d89e832c182 into f843736d5a9a41faa471db9eb4eea69d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:26:28,939 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8dcfd8a4aaa1fef77f3b9d89e832c182: 2024-11-27T07:26:28,939 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., storeName=8dcfd8a4aaa1fef77f3b9d89e832c182/B, priority=13, startTime=1732692388492; duration=0sec 2024-11-27T07:26:28,939 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:26:28,939 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8dcfd8a4aaa1fef77f3b9d89e832c182:B 2024-11-27T07:26:28,939 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:26:28,939 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8dcfd8a4aaa1fef77f3b9d89e832c182/A of 8dcfd8a4aaa1fef77f3b9d89e832c182 into 699e81b22f574df590cbebfb0fe86886(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:26:28,939 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8dcfd8a4aaa1fef77f3b9d89e832c182: 2024-11-27T07:26:28,939 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., storeName=8dcfd8a4aaa1fef77f3b9d89e832c182/A, priority=13, startTime=1732692388492; duration=0sec 2024-11-27T07:26:28,939 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:26:28,939 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8dcfd8a4aaa1fef77f3b9d89e832c182:A 2024-11-27T07:26:28,940 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:26:28,940 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 8dcfd8a4aaa1fef77f3b9d89e832c182/C is initiating minor compaction (all files) 2024-11-27T07:26:28,940 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8dcfd8a4aaa1fef77f3b9d89e832c182/C in TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:28,940 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/d79c221309a94958a6d5207638ae8653, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/d48748f08cb4494eaeba821c76ded266, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/4bca193f361e4334a830f52cb235f638] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp, totalSize=35.2 K 2024-11-27T07:26:28,940 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting d79c221309a94958a6d5207638ae8653, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732692385706 2024-11-27T07:26:28,940 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting d48748f08cb4494eaeba821c76ded266, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732692385755 2024-11-27T07:26:28,941 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 4bca193f361e4334a830f52cb235f638, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732692386926 2024-11-27T07:26:28,946 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8dcfd8a4aaa1fef77f3b9d89e832c182#C#compaction#372 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:26:28,946 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/92b99c42f0654efaa80cbff77dd5ea2c is 50, key is test_row_0/C:col10/1732692387563/Put/seqid=0 2024-11-27T07:26:28,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742274_1450 (size=12104) 2024-11-27T07:26:28,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-27T07:26:29,026 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/2fc98998b96547bbbf3a33964abf3496 2024-11-27T07:26:29,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/cf46b45862e34036b478f1cd2e22ff95 is 50, key is test_row_0/B:col10/1732692387634/Put/seqid=0 2024-11-27T07:26:29,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742275_1451 (size=12001) 2024-11-27T07:26:29,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:29,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34450 deadline: 1732692449119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:29,127 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:29,127 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:29,127 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:29,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:29,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34446 deadline: 1732692449128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:29,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:29,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34398 deadline: 1732692449128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:29,130 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:29,130 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:29,130 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:29,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:29,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:29,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34440 deadline: 1732692449129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:29,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:29,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:29,131 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:29,131 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:29,131 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:29,353 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/92b99c42f0654efaa80cbff77dd5ea2c as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/92b99c42f0654efaa80cbff77dd5ea2c 2024-11-27T07:26:29,357 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8dcfd8a4aaa1fef77f3b9d89e832c182/C of 8dcfd8a4aaa1fef77f3b9d89e832c182 into 92b99c42f0654efaa80cbff77dd5ea2c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:26:29,357 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8dcfd8a4aaa1fef77f3b9d89e832c182: 2024-11-27T07:26:29,357 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., storeName=8dcfd8a4aaa1fef77f3b9d89e832c182/C, priority=13, startTime=1732692388492; duration=0sec 2024-11-27T07:26:29,357 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:26:29,357 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8dcfd8a4aaa1fef77f3b9d89e832c182:C 2024-11-27T07:26:29,435 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/cf46b45862e34036b478f1cd2e22ff95 2024-11-27T07:26:29,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/b40df3e04b034d16a9d718028fd1daaa is 50, key is test_row_0/C:col10/1732692387634/Put/seqid=0 2024-11-27T07:26:29,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:29,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34446 deadline: 1732692449439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:29,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:29,445 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:29,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34398 deadline: 1732692449439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:29,446 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:29,446 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:29,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:29,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:29,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:29,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:29,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34450 deadline: 1732692449439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:29,446 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:29,446 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:29,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:29,446 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:29,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34440 deadline: 1732692449439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:29,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:29,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:29,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:29,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742276_1452 (size=12001) 2024-11-27T07:26:29,447 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/b40df3e04b034d16a9d718028fd1daaa 2024-11-27T07:26:29,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/2fc98998b96547bbbf3a33964abf3496 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/2fc98998b96547bbbf3a33964abf3496 2024-11-27T07:26:29,454 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/2fc98998b96547bbbf3a33964abf3496, entries=150, sequenceid=73, filesize=11.7 K 2024-11-27T07:26:29,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/cf46b45862e34036b478f1cd2e22ff95 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/cf46b45862e34036b478f1cd2e22ff95 2024-11-27T07:26:29,457 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/cf46b45862e34036b478f1cd2e22ff95, entries=150, sequenceid=73, filesize=11.7 K 2024-11-27T07:26:29,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/b40df3e04b034d16a9d718028fd1daaa as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/b40df3e04b034d16a9d718028fd1daaa 2024-11-27T07:26:29,461 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/b40df3e04b034d16a9d718028fd1daaa, entries=150, sequenceid=73, filesize=11.7 K 2024-11-27T07:26:29,462 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 8dcfd8a4aaa1fef77f3b9d89e832c182 in 844ms, sequenceid=73, compaction requested=false 2024-11-27T07:26:29,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 8dcfd8a4aaa1fef77f3b9d89e832c182: 2024-11-27T07:26:29,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:29,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-27T07:26:29,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-27T07:26:29,464 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-27T07:26:29,464 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6110 sec 2024-11-27T07:26:29,466 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.6150 sec 2024-11-27T07:26:29,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 8dcfd8a4aaa1fef77f3b9d89e832c182 2024-11-27T07:26:29,821 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T07:26:29,821 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8dcfd8a4aaa1fef77f3b9d89e832c182 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-27T07:26:29,825 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/b12187803f0042758e34f7dfea64ebdb is 50, key is test_row_0/A:col10/1732692389819/Put/seqid=0 2024-11-27T07:26:29,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742277_1453 (size=14341) 2024-11-27T07:26:29,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:29,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34408 deadline: 1732692449930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:29,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:29,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:29,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:29,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:29,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34446 deadline: 1732692449959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:29,965 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:29,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:29,965 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:29,965 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:29,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34398 deadline: 1732692449959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:29,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:29,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:29,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:29,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:29,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34450 deadline: 1732692449959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:29,966 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:29,966 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:29,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:29,966 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:29,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34440 deadline: 1732692449959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:29,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:29,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:29,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:29,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-27T07:26:29,999 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T07:26:29,999 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T07:26:30,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-27T07:26:30,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-27T07:26:30,001 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T07:26:30,002 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T07:26:30,002 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T07:26:30,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:30,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34408 deadline: 1732692450039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:30,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:30,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:30,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:30,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-27T07:26:30,153 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-27T07:26:30,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:30,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. as already flushing 2024-11-27T07:26:30,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:30,154 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:30,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:30,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:30,228 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/b12187803f0042758e34f7dfea64ebdb 2024-11-27T07:26:30,234 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/111f52b9ef6345399a6b1df2e00f2d56 is 50, key is test_row_0/B:col10/1732692389819/Put/seqid=0 2024-11-27T07:26:30,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742278_1454 (size=12001) 2024-11-27T07:26:30,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:30,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34408 deadline: 1732692450249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:30,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:30,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:30,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:30,306 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-27T07:26:30,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:30,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. as already flushing 2024-11-27T07:26:30,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:30,306 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:30,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:30,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:30,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-27T07:26:30,459 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-27T07:26:30,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:30,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. as already flushing 2024-11-27T07:26:30,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:30,459 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:30,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:30,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:30,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:30,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34408 deadline: 1732692450558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:30,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:30,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:30,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:30,611 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-27T07:26:30,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:30,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. as already flushing 2024-11-27T07:26:30,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:30,612 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:30,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:30,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:30,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-27T07:26:30,638 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/111f52b9ef6345399a6b1df2e00f2d56 2024-11-27T07:26:30,645 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/017b5341b71b4609b3633db75d567263 is 50, key is test_row_0/C:col10/1732692389819/Put/seqid=0 2024-11-27T07:26:30,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742279_1455 (size=12001) 2024-11-27T07:26:30,764 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-27T07:26:30,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:30,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. as already flushing 2024-11-27T07:26:30,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:30,765 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:30,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:30,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:30,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-27T07:26:30,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:30,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. as already flushing 2024-11-27T07:26:30,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:30,917 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:30,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:30,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:30,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:30,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34440 deadline: 1732692450969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:30,976 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 289954c9202d,42117,1732692259671 for 'test_row_1' in 8dcfd8a4aaa1fef77f3b9d89e832c182 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T07:26:30,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:30,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:30,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:30,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:30,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34446 deadline: 1732692450979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:30,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:30,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34398 deadline: 1732692450979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:30,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:30,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34450 deadline: 1732692450979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:30,987 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 289954c9202d,42117,1732692259671 for 'test_row_0' in 8dcfd8a4aaa1fef77f3b9d89e832c182 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T07:26:30,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:30,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:30,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:30,987 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 289954c9202d,42117,1732692259671 for 'test_row_2' in 8dcfd8a4aaa1fef77f3b9d89e832c182 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T07:26:30,988 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:30,988 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:30,988 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:30,987 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 289954c9202d,42117,1732692259671 for 'test_row_0' in 8dcfd8a4aaa1fef77f3b9d89e832c182 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T07:26:30,988 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:30,988 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:30,988 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:31,050 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/017b5341b71b4609b3633db75d567263 2024-11-27T07:26:31,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/b12187803f0042758e34f7dfea64ebdb as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/b12187803f0042758e34f7dfea64ebdb 2024-11-27T07:26:31,057 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/b12187803f0042758e34f7dfea64ebdb, entries=200, sequenceid=91, filesize=14.0 K 2024-11-27T07:26:31,058 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/111f52b9ef6345399a6b1df2e00f2d56 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/111f52b9ef6345399a6b1df2e00f2d56 2024-11-27T07:26:31,061 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/111f52b9ef6345399a6b1df2e00f2d56, entries=150, sequenceid=91, filesize=11.7 K 2024-11-27T07:26:31,061 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/017b5341b71b4609b3633db75d567263 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/017b5341b71b4609b3633db75d567263 2024-11-27T07:26:31,064 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/017b5341b71b4609b3633db75d567263, entries=150, sequenceid=91, filesize=11.7 K 2024-11-27T07:26:31,065 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 8dcfd8a4aaa1fef77f3b9d89e832c182 in 1244ms, sequenceid=91, compaction requested=true 2024-11-27T07:26:31,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8dcfd8a4aaa1fef77f3b9d89e832c182: 2024-11-27T07:26:31,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8dcfd8a4aaa1fef77f3b9d89e832c182:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T07:26:31,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:26:31,065 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:26:31,065 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:26:31,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8dcfd8a4aaa1fef77f3b9d89e832c182:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T07:26:31,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:26:31,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8dcfd8a4aaa1fef77f3b9d89e832c182:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T07:26:31,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:26:31,067 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:26:31,067 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:26:31,067 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 8dcfd8a4aaa1fef77f3b9d89e832c182/B is initiating minor compaction (all files) 2024-11-27T07:26:31,067 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1541): 8dcfd8a4aaa1fef77f3b9d89e832c182/A is initiating minor compaction (all files) 2024-11-27T07:26:31,067 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8dcfd8a4aaa1fef77f3b9d89e832c182/A in TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:31,067 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8dcfd8a4aaa1fef77f3b9d89e832c182/B in TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:31,067 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/699e81b22f574df590cbebfb0fe86886, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/2fc98998b96547bbbf3a33964abf3496, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/b12187803f0042758e34f7dfea64ebdb] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp, totalSize=37.5 K 2024-11-27T07:26:31,067 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/f843736d5a9a41faa471db9eb4eea69d, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/cf46b45862e34036b478f1cd2e22ff95, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/111f52b9ef6345399a6b1df2e00f2d56] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp, totalSize=35.3 K 2024-11-27T07:26:31,067 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 699e81b22f574df590cbebfb0fe86886, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732692386926 2024-11-27T07:26:31,067 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting f843736d5a9a41faa471db9eb4eea69d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732692386926 2024-11-27T07:26:31,067 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting cf46b45862e34036b478f1cd2e22ff95, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1732692387619 2024-11-27T07:26:31,067 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2fc98998b96547bbbf3a33964abf3496, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1732692387619 2024-11-27T07:26:31,068 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting b12187803f0042758e34f7dfea64ebdb, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732692388791 2024-11-27T07:26:31,068 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 111f52b9ef6345399a6b1df2e00f2d56, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732692388791 2024-11-27T07:26:31,070 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-27T07:26:31,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:31,070 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 8dcfd8a4aaa1fef77f3b9d89e832c182 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-27T07:26:31,074 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8dcfd8a4aaa1fef77f3b9d89e832c182#A#compaction#378 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:26:31,074 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8dcfd8a4aaa1fef77f3b9d89e832c182#B#compaction#379 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:26:31,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/8e650a82de704b11a376ea9959bef178 is 50, key is test_row_0/A:col10/1732692389912/Put/seqid=0 2024-11-27T07:26:31,075 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/5975724ed97d42668491574bd8390804 is 50, key is test_row_0/B:col10/1732692389819/Put/seqid=0 2024-11-27T07:26:31,075 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/fdde9133dd894ba9890b2bf717587236 is 50, key is test_row_0/A:col10/1732692389819/Put/seqid=0 2024-11-27T07:26:31,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 8dcfd8a4aaa1fef77f3b9d89e832c182 2024-11-27T07:26:31,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. as already flushing 2024-11-27T07:26:31,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742280_1456 (size=12207) 2024-11-27T07:26:31,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742282_1458 (size=12001) 2024-11-27T07:26:31,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742281_1457 (size=12207) 2024-11-27T07:26:31,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-27T07:26:31,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:31,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34408 deadline: 1732692451157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:31,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:31,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:31,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:31,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:31,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34408 deadline: 1732692451279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:31,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:31,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:31,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:31,483 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/8e650a82de704b11a376ea9959bef178 2024-11-27T07:26:31,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:31,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34408 deadline: 1732692451489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:31,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:31,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:31,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:31,520 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/fdde9133dd894ba9890b2bf717587236 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/fdde9133dd894ba9890b2bf717587236 2024-11-27T07:26:31,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/c5d15ff1a9b84ce0a4194c3ecfb88685 is 50, key is test_row_0/B:col10/1732692389912/Put/seqid=0 2024-11-27T07:26:31,524 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/5975724ed97d42668491574bd8390804 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/5975724ed97d42668491574bd8390804 2024-11-27T07:26:31,528 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8dcfd8a4aaa1fef77f3b9d89e832c182/A of 8dcfd8a4aaa1fef77f3b9d89e832c182 into fdde9133dd894ba9890b2bf717587236(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:26:31,528 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8dcfd8a4aaa1fef77f3b9d89e832c182: 2024-11-27T07:26:31,528 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., storeName=8dcfd8a4aaa1fef77f3b9d89e832c182/A, priority=13, startTime=1732692391065; duration=0sec 2024-11-27T07:26:31,528 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:26:31,528 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8dcfd8a4aaa1fef77f3b9d89e832c182:A 2024-11-27T07:26:31,528 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:26:31,529 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:26:31,529 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1541): 8dcfd8a4aaa1fef77f3b9d89e832c182/C is initiating minor compaction (all files) 2024-11-27T07:26:31,529 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8dcfd8a4aaa1fef77f3b9d89e832c182/C in TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:31,529 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/92b99c42f0654efaa80cbff77dd5ea2c, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/b40df3e04b034d16a9d718028fd1daaa, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/017b5341b71b4609b3633db75d567263] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp, totalSize=35.3 K 2024-11-27T07:26:31,530 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92b99c42f0654efaa80cbff77dd5ea2c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732692386926 2024-11-27T07:26:31,530 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting b40df3e04b034d16a9d718028fd1daaa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1732692387619 2024-11-27T07:26:31,531 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 017b5341b71b4609b3633db75d567263, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732692388791 2024-11-27T07:26:31,531 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8dcfd8a4aaa1fef77f3b9d89e832c182/B of 8dcfd8a4aaa1fef77f3b9d89e832c182 into 5975724ed97d42668491574bd8390804(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:26:31,531 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8dcfd8a4aaa1fef77f3b9d89e832c182: 2024-11-27T07:26:31,531 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., storeName=8dcfd8a4aaa1fef77f3b9d89e832c182/B, priority=13, startTime=1732692391065; duration=0sec 2024-11-27T07:26:31,531 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:26:31,531 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8dcfd8a4aaa1fef77f3b9d89e832c182:B 2024-11-27T07:26:31,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742283_1459 (size=12001) 2024-11-27T07:26:31,533 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/c5d15ff1a9b84ce0a4194c3ecfb88685 2024-11-27T07:26:31,537 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8dcfd8a4aaa1fef77f3b9d89e832c182#C#compaction#382 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:26:31,538 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/accfbc5af49d45d6bb7724020d0635be is 50, key is test_row_0/C:col10/1732692389819/Put/seqid=0 2024-11-27T07:26:31,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/317f7703b9384fa5ac762d8ae87325fa is 50, key is test_row_0/C:col10/1732692389912/Put/seqid=0 2024-11-27T07:26:31,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742285_1461 (size=12001) 2024-11-27T07:26:31,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742284_1460 (size=12207) 2024-11-27T07:26:31,553 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/accfbc5af49d45d6bb7724020d0635be as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/accfbc5af49d45d6bb7724020d0635be 2024-11-27T07:26:31,557 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8dcfd8a4aaa1fef77f3b9d89e832c182/C of 8dcfd8a4aaa1fef77f3b9d89e832c182 into accfbc5af49d45d6bb7724020d0635be(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:26:31,557 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8dcfd8a4aaa1fef77f3b9d89e832c182: 2024-11-27T07:26:31,557 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., storeName=8dcfd8a4aaa1fef77f3b9d89e832c182/C, priority=13, startTime=1732692391065; duration=0sec 2024-11-27T07:26:31,557 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:26:31,557 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8dcfd8a4aaa1fef77f3b9d89e832c182:C 2024-11-27T07:26:31,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:31,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34408 deadline: 1732692451799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:31,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:31,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:31,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:31,949 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/317f7703b9384fa5ac762d8ae87325fa 2024-11-27T07:26:31,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/8e650a82de704b11a376ea9959bef178 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/8e650a82de704b11a376ea9959bef178 2024-11-27T07:26:31,956 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/8e650a82de704b11a376ea9959bef178, entries=150, sequenceid=112, filesize=11.7 K 2024-11-27T07:26:31,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/c5d15ff1a9b84ce0a4194c3ecfb88685 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/c5d15ff1a9b84ce0a4194c3ecfb88685 2024-11-27T07:26:31,960 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/c5d15ff1a9b84ce0a4194c3ecfb88685, entries=150, sequenceid=112, filesize=11.7 K 2024-11-27T07:26:31,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/317f7703b9384fa5ac762d8ae87325fa as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/317f7703b9384fa5ac762d8ae87325fa 2024-11-27T07:26:31,963 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/317f7703b9384fa5ac762d8ae87325fa, entries=150, sequenceid=112, filesize=11.7 K 2024-11-27T07:26:31,964 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 8dcfd8a4aaa1fef77f3b9d89e832c182 in 893ms, sequenceid=112, compaction requested=false 2024-11-27T07:26:31,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 8dcfd8a4aaa1fef77f3b9d89e832c182: 2024-11-27T07:26:31,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:31,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-27T07:26:31,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-27T07:26:31,967 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-27T07:26:31,967 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9630 sec 2024-11-27T07:26:31,968 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.9670 sec 2024-11-27T07:26:32,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-27T07:26:32,149 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T07:26:32,151 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T07:26:32,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-27T07:26:32,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-27T07:26:32,154 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T07:26:32,158 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T07:26:32,158 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T07:26:32,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-27T07:26:32,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-27T07:26:32,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:32,312 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 8dcfd8a4aaa1fef77f3b9d89e832c182 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-27T07:26:32,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 8dcfd8a4aaa1fef77f3b9d89e832c182 2024-11-27T07:26:32,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. as already flushing 2024-11-27T07:26:32,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/d48cd8c45ad0456d841d30d54196295e is 50, key is test_row_0/A:col10/1732692391135/Put/seqid=0 2024-11-27T07:26:32,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742286_1462 (size=12001) 2024-11-27T07:26:32,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:32,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34408 deadline: 1732692452434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:32,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:32,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:32,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:32,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-27T07:26:32,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:32,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34408 deadline: 1732692452549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:32,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:32,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:32,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:32,728 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/d48cd8c45ad0456d841d30d54196295e 2024-11-27T07:26:32,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/5483fafb56e44f9ba9318c857d30453c is 50, key is test_row_0/B:col10/1732692391135/Put/seqid=0 2024-11-27T07:26:32,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742287_1463 (size=12001) 2024-11-27T07:26:32,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:32,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34408 deadline: 1732692452759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:32,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:32,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:32,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:32,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-27T07:26:32,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:32,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34440 deadline: 1732692452989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:32,993 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 289954c9202d,42117,1732692259671 for 'test_row_1' in 8dcfd8a4aaa1fef77f3b9d89e832c182 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T07:26:32,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:32,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:32,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:33,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:33,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34398 deadline: 1732692452999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:33,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:33,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34450 deadline: 1732692452999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:33,002 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 289954c9202d,42117,1732692259671 for 'test_row_0' in 8dcfd8a4aaa1fef77f3b9d89e832c182 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T07:26:33,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:33,002 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 289954c9202d,42117,1732692259671 for 'test_row_2' in 8dcfd8a4aaa1fef77f3b9d89e832c182 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T07:26:33,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:33,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:33,003 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:33,003 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:33,003 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:33,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:33,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34446 deadline: 1732692453009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:33,012 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 289954c9202d,42117,1732692259671 for 'test_row_0' in 8dcfd8a4aaa1fef77f3b9d89e832c182 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T07:26:33,012 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:33,012 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:33,012 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:33,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:33,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34408 deadline: 1732692453079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:33,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:33,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:33,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:33,138 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/5483fafb56e44f9ba9318c857d30453c 2024-11-27T07:26:33,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/5e90acddd7a54870b0be5cfb2db79dd7 is 50, key is test_row_0/C:col10/1732692391135/Put/seqid=0 2024-11-27T07:26:33,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742288_1464 (size=12001) 2024-11-27T07:26:33,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-27T07:26:33,548 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/5e90acddd7a54870b0be5cfb2db79dd7 2024-11-27T07:26:33,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/d48cd8c45ad0456d841d30d54196295e as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/d48cd8c45ad0456d841d30d54196295e 2024-11-27T07:26:33,553 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/d48cd8c45ad0456d841d30d54196295e, entries=150, sequenceid=130, filesize=11.7 K 2024-11-27T07:26:33,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/5483fafb56e44f9ba9318c857d30453c as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/5483fafb56e44f9ba9318c857d30453c 2024-11-27T07:26:33,557 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/5483fafb56e44f9ba9318c857d30453c, entries=150, sequenceid=130, filesize=11.7 K 2024-11-27T07:26:33,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/5e90acddd7a54870b0be5cfb2db79dd7 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/5e90acddd7a54870b0be5cfb2db79dd7 2024-11-27T07:26:33,561 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/5e90acddd7a54870b0be5cfb2db79dd7, entries=150, sequenceid=130, filesize=11.7 K 2024-11-27T07:26:33,561 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 8dcfd8a4aaa1fef77f3b9d89e832c182 in 1249ms, sequenceid=130, compaction requested=true 2024-11-27T07:26:33,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 8dcfd8a4aaa1fef77f3b9d89e832c182: 2024-11-27T07:26:33,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:33,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-27T07:26:33,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-27T07:26:33,564 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-27T07:26:33,564 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4050 sec 2024-11-27T07:26:33,566 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.4140 sec 2024-11-27T07:26:33,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 8dcfd8a4aaa1fef77f3b9d89e832c182 2024-11-27T07:26:33,592 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T07:26:33,593 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8dcfd8a4aaa1fef77f3b9d89e832c182 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-27T07:26:33,596 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/480ddac70a75493182bad3c88e248e4d is 50, key is test_row_0/A:col10/1732692392432/Put/seqid=0 2024-11-27T07:26:33,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742289_1465 (size=14541) 2024-11-27T07:26:33,608 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/480ddac70a75493182bad3c88e248e4d 2024-11-27T07:26:33,614 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/56a631a4ed72405a88b83ccebaf38acc is 50, key is test_row_0/B:col10/1732692392432/Put/seqid=0 2024-11-27T07:26:33,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742290_1466 (size=12151) 2024-11-27T07:26:33,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:33,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34408 deadline: 1732692453669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:33,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:33,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:33,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:33,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:33,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34408 deadline: 1732692453789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:33,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:33,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:33,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:34,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:34,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34408 deadline: 1732692453999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:34,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:34,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:34,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:34,018 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/56a631a4ed72405a88b83ccebaf38acc 2024-11-27T07:26:34,029 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/83024e0ed2694c7182946d4112b1595b is 50, key is test_row_0/C:col10/1732692392432/Put/seqid=0 2024-11-27T07:26:34,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742291_1467 (size=12151) 2024-11-27T07:26:34,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-27T07:26:34,300 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T07:26:34,300 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T07:26:34,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-27T07:26:34,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-27T07:26:34,302 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T07:26:34,302 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T07:26:34,303 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T07:26:34,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:34,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34408 deadline: 1732692454319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:34,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:34,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:34,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:34,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-27T07:26:34,433 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/83024e0ed2694c7182946d4112b1595b 2024-11-27T07:26:34,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/480ddac70a75493182bad3c88e248e4d as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/480ddac70a75493182bad3c88e248e4d 2024-11-27T07:26:34,438 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/480ddac70a75493182bad3c88e248e4d, entries=200, sequenceid=152, filesize=14.2 K 2024-11-27T07:26:34,439 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/56a631a4ed72405a88b83ccebaf38acc as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/56a631a4ed72405a88b83ccebaf38acc 2024-11-27T07:26:34,442 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/56a631a4ed72405a88b83ccebaf38acc, entries=150, sequenceid=152, filesize=11.9 K 2024-11-27T07:26:34,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/83024e0ed2694c7182946d4112b1595b as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/83024e0ed2694c7182946d4112b1595b 2024-11-27T07:26:34,445 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/83024e0ed2694c7182946d4112b1595b, entries=150, sequenceid=152, filesize=11.9 K 2024-11-27T07:26:34,446 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 8dcfd8a4aaa1fef77f3b9d89e832c182 in 854ms, sequenceid=152, compaction requested=true 2024-11-27T07:26:34,446 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8dcfd8a4aaa1fef77f3b9d89e832c182: 2024-11-27T07:26:34,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8dcfd8a4aaa1fef77f3b9d89e832c182:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T07:26:34,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:26:34,446 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T07:26:34,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8dcfd8a4aaa1fef77f3b9d89e832c182:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T07:26:34,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:26:34,446 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T07:26:34,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8dcfd8a4aaa1fef77f3b9d89e832c182:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T07:26:34,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:26:34,447 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T07:26:34,447 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50750 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T07:26:34,447 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 8dcfd8a4aaa1fef77f3b9d89e832c182/B is initiating minor compaction (all files) 2024-11-27T07:26:34,447 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1541): 8dcfd8a4aaa1fef77f3b9d89e832c182/A is initiating minor compaction (all files) 2024-11-27T07:26:34,447 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8dcfd8a4aaa1fef77f3b9d89e832c182/A in TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:34,447 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8dcfd8a4aaa1fef77f3b9d89e832c182/B in TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:34,447 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/5975724ed97d42668491574bd8390804, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/c5d15ff1a9b84ce0a4194c3ecfb88685, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/5483fafb56e44f9ba9318c857d30453c, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/56a631a4ed72405a88b83ccebaf38acc] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp, totalSize=47.2 K 2024-11-27T07:26:34,447 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/fdde9133dd894ba9890b2bf717587236, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/8e650a82de704b11a376ea9959bef178, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/d48cd8c45ad0456d841d30d54196295e, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/480ddac70a75493182bad3c88e248e4d] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp, totalSize=49.6 K 2024-11-27T07:26:34,447 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting fdde9133dd894ba9890b2bf717587236, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732692388791 2024-11-27T07:26:34,447 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 5975724ed97d42668491574bd8390804, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732692388791 2024-11-27T07:26:34,448 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e650a82de704b11a376ea9959bef178, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1732692389912 2024-11-27T07:26:34,448 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting c5d15ff1a9b84ce0a4194c3ecfb88685, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1732692389912 2024-11-27T07:26:34,448 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 5483fafb56e44f9ba9318c857d30453c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732692391121 2024-11-27T07:26:34,448 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting d48cd8c45ad0456d841d30d54196295e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732692391121 2024-11-27T07:26:34,448 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 56a631a4ed72405a88b83ccebaf38acc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732692392419 2024-11-27T07:26:34,448 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 480ddac70a75493182bad3c88e248e4d, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732692392410 2024-11-27T07:26:34,455 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8dcfd8a4aaa1fef77f3b9d89e832c182#A#compaction#390 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:26:34,455 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8dcfd8a4aaa1fef77f3b9d89e832c182#B#compaction#391 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:26:34,455 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/fa69831e760c4516a9ae353194157605 is 50, key is test_row_0/B:col10/1732692392432/Put/seqid=0 2024-11-27T07:26:34,455 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/05076e579605448085ce31a3f1010132 is 50, key is test_row_0/A:col10/1732692392432/Put/seqid=0 2024-11-27T07:26:34,456 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-27T07:26:34,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:34,457 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 8dcfd8a4aaa1fef77f3b9d89e832c182 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-27T07:26:34,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742292_1468 (size=12493) 2024-11-27T07:26:34,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742293_1469 (size=12493) 2024-11-27T07:26:34,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/3fedb318eda44227bc030169856bd112 is 50, key is test_row_0/A:col10/1732692393645/Put/seqid=0 2024-11-27T07:26:34,463 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/fa69831e760c4516a9ae353194157605 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/fa69831e760c4516a9ae353194157605 2024-11-27T07:26:34,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742294_1470 (size=12151) 2024-11-27T07:26:34,469 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8dcfd8a4aaa1fef77f3b9d89e832c182/B of 8dcfd8a4aaa1fef77f3b9d89e832c182 into fa69831e760c4516a9ae353194157605(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:26:34,469 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8dcfd8a4aaa1fef77f3b9d89e832c182: 2024-11-27T07:26:34,469 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., storeName=8dcfd8a4aaa1fef77f3b9d89e832c182/B, priority=12, startTime=1732692394446; duration=0sec 2024-11-27T07:26:34,469 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:26:34,469 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8dcfd8a4aaa1fef77f3b9d89e832c182:B 2024-11-27T07:26:34,469 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T07:26:34,470 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T07:26:34,470 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 8dcfd8a4aaa1fef77f3b9d89e832c182/C is initiating minor compaction (all files) 2024-11-27T07:26:34,470 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8dcfd8a4aaa1fef77f3b9d89e832c182/C in TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:34,470 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/accfbc5af49d45d6bb7724020d0635be, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/317f7703b9384fa5ac762d8ae87325fa, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/5e90acddd7a54870b0be5cfb2db79dd7, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/83024e0ed2694c7182946d4112b1595b] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp, totalSize=47.2 K 2024-11-27T07:26:34,471 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting accfbc5af49d45d6bb7724020d0635be, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732692388791 2024-11-27T07:26:34,471 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 317f7703b9384fa5ac762d8ae87325fa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1732692389912 2024-11-27T07:26:34,471 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e90acddd7a54870b0be5cfb2db79dd7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732692391121 2024-11-27T07:26:34,471 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 83024e0ed2694c7182946d4112b1595b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732692392419 2024-11-27T07:26:34,479 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8dcfd8a4aaa1fef77f3b9d89e832c182#C#compaction#393 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:26:34,479 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/ba4dba4ef3ce47929c6f3868d96fe051 is 50, key is test_row_0/C:col10/1732692392432/Put/seqid=0 2024-11-27T07:26:34,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742295_1471 (size=12493) 2024-11-27T07:26:34,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-27T07:26:34,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. as already flushing 2024-11-27T07:26:34,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 8dcfd8a4aaa1fef77f3b9d89e832c182 2024-11-27T07:26:34,863 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/05076e579605448085ce31a3f1010132 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/05076e579605448085ce31a3f1010132 2024-11-27T07:26:34,866 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/3fedb318eda44227bc030169856bd112 2024-11-27T07:26:34,867 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8dcfd8a4aaa1fef77f3b9d89e832c182/A of 8dcfd8a4aaa1fef77f3b9d89e832c182 into 05076e579605448085ce31a3f1010132(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:26:34,867 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8dcfd8a4aaa1fef77f3b9d89e832c182: 2024-11-27T07:26:34,867 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., storeName=8dcfd8a4aaa1fef77f3b9d89e832c182/A, priority=12, startTime=1732692394446; duration=0sec 2024-11-27T07:26:34,867 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:26:34,867 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8dcfd8a4aaa1fef77f3b9d89e832c182:A 2024-11-27T07:26:34,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/b4368286e31949ea90f49b244938b68f is 50, key is test_row_0/B:col10/1732692393645/Put/seqid=0 2024-11-27T07:26:34,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742296_1472 (size=12151) 2024-11-27T07:26:34,887 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/ba4dba4ef3ce47929c6f3868d96fe051 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/ba4dba4ef3ce47929c6f3868d96fe051 2024-11-27T07:26:34,891 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8dcfd8a4aaa1fef77f3b9d89e832c182/C of 8dcfd8a4aaa1fef77f3b9d89e832c182 into ba4dba4ef3ce47929c6f3868d96fe051(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:26:34,891 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8dcfd8a4aaa1fef77f3b9d89e832c182: 2024-11-27T07:26:34,891 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., storeName=8dcfd8a4aaa1fef77f3b9d89e832c182/C, priority=12, startTime=1732692394446; duration=0sec 2024-11-27T07:26:34,891 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:26:34,891 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8dcfd8a4aaa1fef77f3b9d89e832c182:C 2024-11-27T07:26:34,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-27T07:26:34,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:34,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34408 deadline: 1732692454929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:34,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:34,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:34,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:35,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:35,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34408 deadline: 1732692455039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:35,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:35,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:35,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:35,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:35,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34408 deadline: 1732692455249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:35,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:35,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:35,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:35,276 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/b4368286e31949ea90f49b244938b68f 2024-11-27T07:26:35,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/db9e40a6b86f46e09863674589cae253 is 50, key is test_row_0/C:col10/1732692393645/Put/seqid=0 2024-11-27T07:26:35,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742297_1473 (size=12151) 2024-11-27T07:26:35,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-27T07:26:35,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:35,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34408 deadline: 1732692455559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:35,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:35,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:35,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:35,685 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/db9e40a6b86f46e09863674589cae253 2024-11-27T07:26:35,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/3fedb318eda44227bc030169856bd112 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/3fedb318eda44227bc030169856bd112 2024-11-27T07:26:35,694 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/3fedb318eda44227bc030169856bd112, entries=150, sequenceid=166, filesize=11.9 K 2024-11-27T07:26:35,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/b4368286e31949ea90f49b244938b68f as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/b4368286e31949ea90f49b244938b68f 2024-11-27T07:26:35,698 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/b4368286e31949ea90f49b244938b68f, entries=150, sequenceid=166, filesize=11.9 K 2024-11-27T07:26:35,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/db9e40a6b86f46e09863674589cae253 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/db9e40a6b86f46e09863674589cae253 2024-11-27T07:26:35,703 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/db9e40a6b86f46e09863674589cae253, entries=150, sequenceid=166, filesize=11.9 K 2024-11-27T07:26:35,703 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 8dcfd8a4aaa1fef77f3b9d89e832c182 in 1246ms, sequenceid=166, compaction requested=false 2024-11-27T07:26:35,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 8dcfd8a4aaa1fef77f3b9d89e832c182: 2024-11-27T07:26:35,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:35,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-27T07:26:35,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-27T07:26:35,707 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-27T07:26:35,707 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4030 sec 2024-11-27T07:26:35,708 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.4070 sec 2024-11-27T07:26:36,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 8dcfd8a4aaa1fef77f3b9d89e832c182 2024-11-27T07:26:36,083 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T07:26:36,083 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8dcfd8a4aaa1fef77f3b9d89e832c182 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-27T07:26:36,087 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/82390dac06e94ccf802eef8c9acd77c1 is 50, key is test_row_0/A:col10/1732692394918/Put/seqid=0 2024-11-27T07:26:36,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742298_1474 (size=14541) 2024-11-27T07:26:36,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:36,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34408 deadline: 1732692456136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:36,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:36,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:36,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:36,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:36,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34408 deadline: 1732692456249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:36,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:36,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:36,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:36,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-27T07:26:36,459 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T07:26:36,459 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T07:26:36,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-27T07:26:36,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-27T07:26:36,461 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T07:26:36,461 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T07:26:36,462 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T07:26:36,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:36,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34408 deadline: 1732692456459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:36,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:36,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:36,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:36,490 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/82390dac06e94ccf802eef8c9acd77c1 2024-11-27T07:26:36,540 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/4fed3a9b99814a36a620fbb231f25776 is 50, key is test_row_0/B:col10/1732692394918/Put/seqid=0 2024-11-27T07:26:36,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742299_1475 (size=12151) 2024-11-27T07:26:36,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-27T07:26:36,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-27T07:26:36,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:36,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. as already flushing 2024-11-27T07:26:36,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:36,614 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:36,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:36,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:36,766 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-27T07:26:36,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:36,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. as already flushing 2024-11-27T07:26:36,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:36,766 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:36,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:36,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:36,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:36,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34408 deadline: 1732692456769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:36,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:36,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:36,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:36,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-27T07:26:36,919 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-27T07:26:36,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:36,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. as already flushing 2024-11-27T07:26:36,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:36,919 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:36,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:36,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:36,943 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/4fed3a9b99814a36a620fbb231f25776 2024-11-27T07:26:36,950 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/b97998b68a564d45bf825a28ba090cef is 50, key is test_row_0/C:col10/1732692394918/Put/seqid=0 2024-11-27T07:26:36,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742300_1476 (size=12151) 2024-11-27T07:26:37,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:37,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34398 deadline: 1732692457019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:37,024 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 289954c9202d,42117,1732692259671 for 'test_row_0' in 8dcfd8a4aaa1fef77f3b9d89e832c182 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T07:26:37,024 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:37,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:37,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:37,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:37,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34440 deadline: 1732692457029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:37,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:37,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34450 deadline: 1732692457029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:37,036 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 289954c9202d,42117,1732692259671 for 'test_row_1' in 8dcfd8a4aaa1fef77f3b9d89e832c182 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8231 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T07:26:37,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:37,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:37,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:37,037 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 289954c9202d,42117,1732692259671 for 'test_row_2' in 8dcfd8a4aaa1fef77f3b9d89e832c182 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8238 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T07:26:37,037 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:37,037 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:37,037 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:37,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:37,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34446 deadline: 1732692457049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:37,056 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 289954c9202d,42117,1732692259671 for 'test_row_0' in 8dcfd8a4aaa1fef77f3b9d89e832c182 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8252 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T07:26:37,056 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:37,057 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:37,057 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:37,071 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-27T07:26:37,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:37,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. as already flushing 2024-11-27T07:26:37,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:37,071 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:37,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:37,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:37,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-27T07:26:37,224 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-27T07:26:37,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:37,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. as already flushing 2024-11-27T07:26:37,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:37,224 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:37,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:37,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:37,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:37,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34408 deadline: 1732692457289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:37,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:37,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:37,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:37,353 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/b97998b68a564d45bf825a28ba090cef 2024-11-27T07:26:37,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/82390dac06e94ccf802eef8c9acd77c1 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/82390dac06e94ccf802eef8c9acd77c1 2024-11-27T07:26:37,359 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/82390dac06e94ccf802eef8c9acd77c1, entries=200, sequenceid=192, filesize=14.2 K 2024-11-27T07:26:37,359 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/4fed3a9b99814a36a620fbb231f25776 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/4fed3a9b99814a36a620fbb231f25776 2024-11-27T07:26:37,362 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/4fed3a9b99814a36a620fbb231f25776, entries=150, sequenceid=192, filesize=11.9 K 2024-11-27T07:26:37,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/b97998b68a564d45bf825a28ba090cef as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/b97998b68a564d45bf825a28ba090cef 2024-11-27T07:26:37,366 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/b97998b68a564d45bf825a28ba090cef, entries=150, sequenceid=192, filesize=11.9 K 2024-11-27T07:26:37,366 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 8dcfd8a4aaa1fef77f3b9d89e832c182 in 1283ms, sequenceid=192, compaction requested=true 2024-11-27T07:26:37,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8dcfd8a4aaa1fef77f3b9d89e832c182: 2024-11-27T07:26:37,367 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:26:37,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8dcfd8a4aaa1fef77f3b9d89e832c182:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T07:26:37,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:26:37,367 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:26:37,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8dcfd8a4aaa1fef77f3b9d89e832c182:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T07:26:37,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:26:37,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8dcfd8a4aaa1fef77f3b9d89e832c182:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T07:26:37,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:26:37,368 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:26:37,368 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1541): 8dcfd8a4aaa1fef77f3b9d89e832c182/A is initiating minor compaction (all files) 2024-11-27T07:26:37,368 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:26:37,368 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8dcfd8a4aaa1fef77f3b9d89e832c182/A in TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:37,368 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 8dcfd8a4aaa1fef77f3b9d89e832c182/B is initiating minor compaction (all files) 2024-11-27T07:26:37,368 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8dcfd8a4aaa1fef77f3b9d89e832c182/B in TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:37,368 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/05076e579605448085ce31a3f1010132, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/3fedb318eda44227bc030169856bd112, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/82390dac06e94ccf802eef8c9acd77c1] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp, totalSize=38.3 K 2024-11-27T07:26:37,368 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/fa69831e760c4516a9ae353194157605, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/b4368286e31949ea90f49b244938b68f, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/4fed3a9b99814a36a620fbb231f25776] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp, totalSize=35.9 K 2024-11-27T07:26:37,368 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05076e579605448085ce31a3f1010132, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732692392419 2024-11-27T07:26:37,368 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting fa69831e760c4516a9ae353194157605, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732692392419 2024-11-27T07:26:37,368 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting b4368286e31949ea90f49b244938b68f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732692393618 2024-11-27T07:26:37,369 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3fedb318eda44227bc030169856bd112, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732692393618 2024-11-27T07:26:37,369 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fed3a9b99814a36a620fbb231f25776, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732692394918 2024-11-27T07:26:37,369 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82390dac06e94ccf802eef8c9acd77c1, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732692394857 2024-11-27T07:26:37,374 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8dcfd8a4aaa1fef77f3b9d89e832c182#A#compaction#399 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:26:37,375 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/3f6022809d6f4a0fb54b1fbec3c0dc5b is 50, key is test_row_0/A:col10/1732692394918/Put/seqid=0 2024-11-27T07:26:37,375 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8dcfd8a4aaa1fef77f3b9d89e832c182#B#compaction#400 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:26:37,375 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/6a69ad383a5947cc9c8a6e7f7d3b63c5 is 50, key is test_row_0/B:col10/1732692394918/Put/seqid=0 2024-11-27T07:26:37,377 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-27T07:26:37,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:37,377 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 8dcfd8a4aaa1fef77f3b9d89e832c182 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-27T07:26:37,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742301_1477 (size=12595) 2024-11-27T07:26:37,390 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/3f6022809d6f4a0fb54b1fbec3c0dc5b as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/3f6022809d6f4a0fb54b1fbec3c0dc5b 2024-11-27T07:26:37,396 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8dcfd8a4aaa1fef77f3b9d89e832c182/A of 8dcfd8a4aaa1fef77f3b9d89e832c182 into 3f6022809d6f4a0fb54b1fbec3c0dc5b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:26:37,396 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8dcfd8a4aaa1fef77f3b9d89e832c182: 2024-11-27T07:26:37,396 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., storeName=8dcfd8a4aaa1fef77f3b9d89e832c182/A, priority=13, startTime=1732692397367; duration=0sec 2024-11-27T07:26:37,396 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:26:37,396 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8dcfd8a4aaa1fef77f3b9d89e832c182:A 2024-11-27T07:26:37,396 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:26:37,397 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:26:37,397 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1541): 8dcfd8a4aaa1fef77f3b9d89e832c182/C is initiating minor compaction (all files) 2024-11-27T07:26:37,397 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8dcfd8a4aaa1fef77f3b9d89e832c182/C in TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:37,397 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/ba4dba4ef3ce47929c6f3868d96fe051, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/db9e40a6b86f46e09863674589cae253, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/b97998b68a564d45bf825a28ba090cef] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp, totalSize=35.9 K 2024-11-27T07:26:37,397 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba4dba4ef3ce47929c6f3868d96fe051, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732692392419 2024-11-27T07:26:37,398 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting db9e40a6b86f46e09863674589cae253, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732692393618 2024-11-27T07:26:37,398 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting b97998b68a564d45bf825a28ba090cef, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732692394918 2024-11-27T07:26:37,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/f391d6167fc04954a08a3e1e8f6c82cf is 50, key is test_row_0/A:col10/1732692396118/Put/seqid=0 2024-11-27T07:26:37,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742302_1478 (size=12595) 2024-11-27T07:26:37,408 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/6a69ad383a5947cc9c8a6e7f7d3b63c5 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/6a69ad383a5947cc9c8a6e7f7d3b63c5 2024-11-27T07:26:37,412 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8dcfd8a4aaa1fef77f3b9d89e832c182/B of 8dcfd8a4aaa1fef77f3b9d89e832c182 into 6a69ad383a5947cc9c8a6e7f7d3b63c5(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:26:37,412 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8dcfd8a4aaa1fef77f3b9d89e832c182: 2024-11-27T07:26:37,412 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., storeName=8dcfd8a4aaa1fef77f3b9d89e832c182/B, priority=13, startTime=1732692397367; duration=0sec 2024-11-27T07:26:37,412 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:26:37,412 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8dcfd8a4aaa1fef77f3b9d89e832c182:B 2024-11-27T07:26:37,423 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8dcfd8a4aaa1fef77f3b9d89e832c182#C#compaction#402 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:26:37,424 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/5ef85c01816f4cdd9b05c45d0eae7d86 is 50, key is test_row_0/C:col10/1732692394918/Put/seqid=0 2024-11-27T07:26:37,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742303_1479 (size=12151) 2024-11-27T07:26:37,425 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/f391d6167fc04954a08a3e1e8f6c82cf 2024-11-27T07:26:37,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742304_1480 (size=12595) 2024-11-27T07:26:37,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/061a51660d3c464391186767d83f3f2f is 50, key is test_row_0/B:col10/1732692396118/Put/seqid=0 2024-11-27T07:26:37,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742305_1481 (size=12151) 2024-11-27T07:26:37,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-27T07:26:37,831 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/5ef85c01816f4cdd9b05c45d0eae7d86 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/5ef85c01816f4cdd9b05c45d0eae7d86 2024-11-27T07:26:37,834 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8dcfd8a4aaa1fef77f3b9d89e832c182/C of 8dcfd8a4aaa1fef77f3b9d89e832c182 into 5ef85c01816f4cdd9b05c45d0eae7d86(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:26:37,834 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8dcfd8a4aaa1fef77f3b9d89e832c182: 2024-11-27T07:26:37,834 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., storeName=8dcfd8a4aaa1fef77f3b9d89e832c182/C, priority=13, startTime=1732692397367; duration=0sec 2024-11-27T07:26:37,834 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:26:37,834 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8dcfd8a4aaa1fef77f3b9d89e832c182:C 2024-11-27T07:26:37,836 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/061a51660d3c464391186767d83f3f2f 2024-11-27T07:26:37,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/bd04e02b74d94f1abf5994b363da24c0 is 50, key is test_row_0/C:col10/1732692396118/Put/seqid=0 2024-11-27T07:26:37,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742306_1482 (size=12151) 2024-11-27T07:26:37,847 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/bd04e02b74d94f1abf5994b363da24c0 2024-11-27T07:26:37,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/f391d6167fc04954a08a3e1e8f6c82cf as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/f391d6167fc04954a08a3e1e8f6c82cf 2024-11-27T07:26:37,855 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/f391d6167fc04954a08a3e1e8f6c82cf, entries=150, sequenceid=205, filesize=11.9 K 2024-11-27T07:26:37,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/061a51660d3c464391186767d83f3f2f as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/061a51660d3c464391186767d83f3f2f 2024-11-27T07:26:37,861 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/061a51660d3c464391186767d83f3f2f, entries=150, sequenceid=205, filesize=11.9 K 2024-11-27T07:26:37,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/bd04e02b74d94f1abf5994b363da24c0 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/bd04e02b74d94f1abf5994b363da24c0 2024-11-27T07:26:37,865 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/bd04e02b74d94f1abf5994b363da24c0, entries=150, sequenceid=205, filesize=11.9 K 2024-11-27T07:26:37,866 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=0 B/0 for 8dcfd8a4aaa1fef77f3b9d89e832c182 in 489ms, sequenceid=205, compaction requested=false 2024-11-27T07:26:37,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 8dcfd8a4aaa1fef77f3b9d89e832c182: 2024-11-27T07:26:37,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:37,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-27T07:26:37,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-27T07:26:37,871 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-27T07:26:37,871 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4070 sec 2024-11-27T07:26:37,873 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.4130 sec 2024-11-27T07:26:38,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 8dcfd8a4aaa1fef77f3b9d89e832c182 2024-11-27T07:26:38,326 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T07:26:38,326 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8dcfd8a4aaa1fef77f3b9d89e832c182 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-27T07:26:38,330 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/16f21f9201d4431a9e93111cacf861ea is 50, key is test_row_0/A:col10/1732692398315/Put/seqid=0 2024-11-27T07:26:38,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742307_1483 (size=14541) 2024-11-27T07:26:38,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:38,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34408 deadline: 1732692458441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:38,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:38,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:38,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:38,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:38,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34408 deadline: 1732692458549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:38,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:38,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:38,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:38,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-27T07:26:38,609 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T07:26:38,609 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T07:26:38,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-27T07:26:38,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-27T07:26:38,611 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T07:26:38,611 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T07:26:38,611 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T07:26:38,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-27T07:26:38,733 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/16f21f9201d4431a9e93111cacf861ea 2024-11-27T07:26:38,738 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/b7cbfdfade8b46938e8bced49133ae05 is 50, key is test_row_0/B:col10/1732692398315/Put/seqid=0 2024-11-27T07:26:38,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742308_1484 (size=12151) 2024-11-27T07:26:38,763 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-27T07:26:38,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:38,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. as already flushing 2024-11-27T07:26:38,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:38,763 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:38,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:38,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:38,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:38,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34408 deadline: 1732692458759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:38,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:38,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:38,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:38,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-27T07:26:38,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:38,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. as already flushing 2024-11-27T07:26:38,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:38,916 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:38,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:38,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:38,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-27T07:26:39,068 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-27T07:26:39,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:39,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. as already flushing 2024-11-27T07:26:39,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:39,069 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:39,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:39,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:39,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:39,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34408 deadline: 1732692459079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:39,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:39,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:39,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:39,141 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/b7cbfdfade8b46938e8bced49133ae05 2024-11-27T07:26:39,147 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/5ea27d631f5645b98fbcb87b0261bc2f is 50, key is test_row_0/C:col10/1732692398315/Put/seqid=0 2024-11-27T07:26:39,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742309_1485 (size=12151) 2024-11-27T07:26:39,221 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-27T07:26:39,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:39,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. as already flushing 2024-11-27T07:26:39,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:39,221 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:39,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:39,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:39,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-27T07:26:39,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-27T07:26:39,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:39,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. as already flushing 2024-11-27T07:26:39,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:39,374 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:39,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:39,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:39,526 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-27T07:26:39,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:39,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. as already flushing 2024-11-27T07:26:39,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:39,527 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:39,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:39,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:39,551 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/5ea27d631f5645b98fbcb87b0261bc2f 2024-11-27T07:26:39,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/16f21f9201d4431a9e93111cacf861ea as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/16f21f9201d4431a9e93111cacf861ea 2024-11-27T07:26:39,557 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/16f21f9201d4431a9e93111cacf861ea, entries=200, sequenceid=219, filesize=14.2 K 2024-11-27T07:26:39,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/b7cbfdfade8b46938e8bced49133ae05 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/b7cbfdfade8b46938e8bced49133ae05 2024-11-27T07:26:39,559 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/b7cbfdfade8b46938e8bced49133ae05, entries=150, sequenceid=219, filesize=11.9 K 2024-11-27T07:26:39,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/5ea27d631f5645b98fbcb87b0261bc2f as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/5ea27d631f5645b98fbcb87b0261bc2f 2024-11-27T07:26:39,562 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/5ea27d631f5645b98fbcb87b0261bc2f, entries=150, sequenceid=219, filesize=11.9 K 2024-11-27T07:26:39,563 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 8dcfd8a4aaa1fef77f3b9d89e832c182 in 1237ms, sequenceid=219, compaction requested=true 2024-11-27T07:26:39,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8dcfd8a4aaa1fef77f3b9d89e832c182: 2024-11-27T07:26:39,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8dcfd8a4aaa1fef77f3b9d89e832c182:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T07:26:39,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:26:39,563 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:26:39,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8dcfd8a4aaa1fef77f3b9d89e832c182:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T07:26:39,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:26:39,563 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:26:39,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8dcfd8a4aaa1fef77f3b9d89e832c182:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T07:26:39,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:26:39,564 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:26:39,564 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39287 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:26:39,564 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 8dcfd8a4aaa1fef77f3b9d89e832c182/B is initiating minor compaction (all files) 2024-11-27T07:26:39,564 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1541): 8dcfd8a4aaa1fef77f3b9d89e832c182/A is initiating minor compaction (all files) 2024-11-27T07:26:39,564 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8dcfd8a4aaa1fef77f3b9d89e832c182/B in TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:39,564 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8dcfd8a4aaa1fef77f3b9d89e832c182/A in TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:39,564 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/3f6022809d6f4a0fb54b1fbec3c0dc5b, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/f391d6167fc04954a08a3e1e8f6c82cf, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/16f21f9201d4431a9e93111cacf861ea] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp, totalSize=38.4 K 2024-11-27T07:26:39,564 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/6a69ad383a5947cc9c8a6e7f7d3b63c5, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/061a51660d3c464391186767d83f3f2f, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/b7cbfdfade8b46938e8bced49133ae05] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp, totalSize=36.0 K 2024-11-27T07:26:39,564 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f6022809d6f4a0fb54b1fbec3c0dc5b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732692394918 2024-11-27T07:26:39,564 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a69ad383a5947cc9c8a6e7f7d3b63c5, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732692394918 2024-11-27T07:26:39,564 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting f391d6167fc04954a08a3e1e8f6c82cf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1732692396118 2024-11-27T07:26:39,564 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 061a51660d3c464391186767d83f3f2f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1732692396118 2024-11-27T07:26:39,565 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16f21f9201d4431a9e93111cacf861ea, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1732692398314 2024-11-27T07:26:39,565 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting b7cbfdfade8b46938e8bced49133ae05, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1732692398315 2024-11-27T07:26:39,571 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8dcfd8a4aaa1fef77f3b9d89e832c182#B#compaction#408 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:26:39,571 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/062e63c25ecd4051a75980bce05e0bef is 50, key is test_row_0/B:col10/1732692398315/Put/seqid=0 2024-11-27T07:26:39,574 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8dcfd8a4aaa1fef77f3b9d89e832c182#A#compaction#409 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:26:39,574 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/c192f94b8f3a4f2fa615c3c2c3ad9394 is 50, key is test_row_0/A:col10/1732692398315/Put/seqid=0 2024-11-27T07:26:39,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742310_1486 (size=12697) 2024-11-27T07:26:39,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742311_1487 (size=12697) 2024-11-27T07:26:39,584 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/c192f94b8f3a4f2fa615c3c2c3ad9394 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/c192f94b8f3a4f2fa615c3c2c3ad9394 2024-11-27T07:26:39,588 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8dcfd8a4aaa1fef77f3b9d89e832c182/A of 8dcfd8a4aaa1fef77f3b9d89e832c182 into c192f94b8f3a4f2fa615c3c2c3ad9394(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:26:39,588 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8dcfd8a4aaa1fef77f3b9d89e832c182: 2024-11-27T07:26:39,588 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., storeName=8dcfd8a4aaa1fef77f3b9d89e832c182/A, priority=13, startTime=1732692399563; duration=0sec 2024-11-27T07:26:39,588 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:26:39,588 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8dcfd8a4aaa1fef77f3b9d89e832c182:A 2024-11-27T07:26:39,588 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:26:39,589 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:26:39,589 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1541): 8dcfd8a4aaa1fef77f3b9d89e832c182/C is initiating minor compaction (all files) 2024-11-27T07:26:39,589 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8dcfd8a4aaa1fef77f3b9d89e832c182/C in TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:39,589 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/5ef85c01816f4cdd9b05c45d0eae7d86, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/bd04e02b74d94f1abf5994b363da24c0, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/5ea27d631f5645b98fbcb87b0261bc2f] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp, totalSize=36.0 K 2024-11-27T07:26:39,590 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ef85c01816f4cdd9b05c45d0eae7d86, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732692394918 2024-11-27T07:26:39,590 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd04e02b74d94f1abf5994b363da24c0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1732692396118 2024-11-27T07:26:39,590 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ea27d631f5645b98fbcb87b0261bc2f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1732692398315 2024-11-27T07:26:39,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 8dcfd8a4aaa1fef77f3b9d89e832c182 2024-11-27T07:26:39,593 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T07:26:39,593 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8dcfd8a4aaa1fef77f3b9d89e832c182 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-27T07:26:39,600 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/12c764db72b244748846b3c0cafd8e84 is 50, key is test_row_0/A:col10/1732692398440/Put/seqid=0 2024-11-27T07:26:39,610 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8dcfd8a4aaa1fef77f3b9d89e832c182#C#compaction#411 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:26:39,610 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/ae279d3f21934f70bbf03b2ae9dfec56 is 50, key is test_row_0/C:col10/1732692398315/Put/seqid=0 2024-11-27T07:26:39,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742312_1488 (size=14541) 2024-11-27T07:26:39,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742313_1489 (size=12697) 2024-11-27T07:26:39,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:39,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34408 deadline: 1732692459634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:39,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:39,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:39,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:39,679 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-27T07:26:39,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:39,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. as already flushing 2024-11-27T07:26:39,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:39,679 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:39,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:39,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:39,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-27T07:26:39,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:39,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34408 deadline: 1732692459749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:39,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:39,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:39,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:39,832 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-27T07:26:39,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:39,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. as already flushing 2024-11-27T07:26:39,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:39,832 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:39,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:39,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:39,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:39,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34408 deadline: 1732692459958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:39,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:39,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:39,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:39,980 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/062e63c25ecd4051a75980bce05e0bef as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/062e63c25ecd4051a75980bce05e0bef 2024-11-27T07:26:39,984 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8dcfd8a4aaa1fef77f3b9d89e832c182/B of 8dcfd8a4aaa1fef77f3b9d89e832c182 into 062e63c25ecd4051a75980bce05e0bef(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:26:39,984 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-27T07:26:39,984 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8dcfd8a4aaa1fef77f3b9d89e832c182: 2024-11-27T07:26:39,984 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., storeName=8dcfd8a4aaa1fef77f3b9d89e832c182/B, priority=13, startTime=1732692399563; duration=0sec 2024-11-27T07:26:39,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:39,985 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:26:39,985 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8dcfd8a4aaa1fef77f3b9d89e832c182:B 2024-11-27T07:26:39,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. as already flushing 2024-11-27T07:26:39,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:39,985 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:39,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:39,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:40,024 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/12c764db72b244748846b3c0cafd8e84 2024-11-27T07:26:40,028 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/ae279d3f21934f70bbf03b2ae9dfec56 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/ae279d3f21934f70bbf03b2ae9dfec56 2024-11-27T07:26:40,030 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/d090e1c68bfe4db2bcf8475a092304ca is 50, key is test_row_0/B:col10/1732692398440/Put/seqid=0 2024-11-27T07:26:40,031 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8dcfd8a4aaa1fef77f3b9d89e832c182/C of 8dcfd8a4aaa1fef77f3b9d89e832c182 into ae279d3f21934f70bbf03b2ae9dfec56(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:26:40,031 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8dcfd8a4aaa1fef77f3b9d89e832c182: 2024-11-27T07:26:40,031 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., storeName=8dcfd8a4aaa1fef77f3b9d89e832c182/C, priority=13, startTime=1732692399563; duration=0sec 2024-11-27T07:26:40,031 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:26:40,031 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8dcfd8a4aaa1fef77f3b9d89e832c182:C 2024-11-27T07:26:40,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742314_1490 (size=12151) 2024-11-27T07:26:40,034 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/d090e1c68bfe4db2bcf8475a092304ca 2024-11-27T07:26:40,039 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/b8852b29f5764e5dbb50b00879434794 is 50, key is test_row_0/C:col10/1732692398440/Put/seqid=0 2024-11-27T07:26:40,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742315_1491 (size=12151) 2024-11-27T07:26:40,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-27T07:26:40,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:40,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. as already flushing 2024-11-27T07:26:40,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:40,137 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:40,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:40,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:40,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:40,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34408 deadline: 1732692460279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:40,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:40,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:40,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:40,289 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-27T07:26:40,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:40,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. as already flushing 2024-11-27T07:26:40,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:40,290 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:40,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:40,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:40,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-27T07:26:40,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:40,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. as already flushing 2024-11-27T07:26:40,442 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/b8852b29f5764e5dbb50b00879434794 2024-11-27T07:26:40,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:40,442 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:40,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:40,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:40,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/12c764db72b244748846b3c0cafd8e84 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/12c764db72b244748846b3c0cafd8e84 2024-11-27T07:26:40,448 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/12c764db72b244748846b3c0cafd8e84, entries=200, sequenceid=246, filesize=14.2 K 2024-11-27T07:26:40,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/d090e1c68bfe4db2bcf8475a092304ca as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/d090e1c68bfe4db2bcf8475a092304ca 2024-11-27T07:26:40,451 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/d090e1c68bfe4db2bcf8475a092304ca, entries=150, sequenceid=246, filesize=11.9 K 2024-11-27T07:26:40,452 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/b8852b29f5764e5dbb50b00879434794 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/b8852b29f5764e5dbb50b00879434794 2024-11-27T07:26:40,455 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/b8852b29f5764e5dbb50b00879434794, entries=150, sequenceid=246, filesize=11.9 K 2024-11-27T07:26:40,456 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 8dcfd8a4aaa1fef77f3b9d89e832c182 in 862ms, sequenceid=246, compaction requested=false 2024-11-27T07:26:40,456 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8dcfd8a4aaa1fef77f3b9d89e832c182: 2024-11-27T07:26:40,594 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-27T07:26:40,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:40,595 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 8dcfd8a4aaa1fef77f3b9d89e832c182 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-27T07:26:40,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/ae3e2fe889de4c4d929e0be900cd1061 is 50, key is test_row_0/A:col10/1732692399633/Put/seqid=0 2024-11-27T07:26:40,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742316_1492 (size=12151) 2024-11-27T07:26:40,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-27T07:26:40,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 8dcfd8a4aaa1fef77f3b9d89e832c182 2024-11-27T07:26:40,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. as already flushing 2024-11-27T07:26:40,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:40,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34408 deadline: 1732692460915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:40,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:40,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:40,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:41,001 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/ae3e2fe889de4c4d929e0be900cd1061 2024-11-27T07:26:41,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/2b6cba7a5e0143029138eaef0ee629e1 is 50, key is test_row_0/B:col10/1732692399633/Put/seqid=0 2024-11-27T07:26:41,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742317_1493 (size=12151) 2024-11-27T07:26:41,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:41,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34408 deadline: 1732692461028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:41,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:41,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:41,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:41,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:41,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34408 deadline: 1732692461239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:41,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:41,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:41,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:41,409 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/2b6cba7a5e0143029138eaef0ee629e1 2024-11-27T07:26:41,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/4db144ec6fb2410a86d5533307cf0cbb is 50, key is test_row_0/C:col10/1732692399633/Put/seqid=0 2024-11-27T07:26:41,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742318_1494 (size=12151) 2024-11-27T07:26:41,418 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/4db144ec6fb2410a86d5533307cf0cbb 2024-11-27T07:26:41,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/ae3e2fe889de4c4d929e0be900cd1061 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/ae3e2fe889de4c4d929e0be900cd1061 2024-11-27T07:26:41,424 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/ae3e2fe889de4c4d929e0be900cd1061, entries=150, sequenceid=258, filesize=11.9 K 2024-11-27T07:26:41,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/2b6cba7a5e0143029138eaef0ee629e1 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/2b6cba7a5e0143029138eaef0ee629e1 2024-11-27T07:26:41,427 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/2b6cba7a5e0143029138eaef0ee629e1, entries=150, sequenceid=258, filesize=11.9 K 2024-11-27T07:26:41,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/4db144ec6fb2410a86d5533307cf0cbb as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/4db144ec6fb2410a86d5533307cf0cbb 2024-11-27T07:26:41,430 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/4db144ec6fb2410a86d5533307cf0cbb, entries=150, sequenceid=258, filesize=11.9 K 2024-11-27T07:26:41,431 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 8dcfd8a4aaa1fef77f3b9d89e832c182 in 836ms, sequenceid=258, compaction requested=true 2024-11-27T07:26:41,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 8dcfd8a4aaa1fef77f3b9d89e832c182: 2024-11-27T07:26:41,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:41,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-27T07:26:41,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-27T07:26:41,434 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-27T07:26:41,434 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8210 sec 2024-11-27T07:26:41,435 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 2.8250 sec 2024-11-27T07:26:41,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 8dcfd8a4aaa1fef77f3b9d89e832c182 2024-11-27T07:26:41,554 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T07:26:41,554 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8dcfd8a4aaa1fef77f3b9d89e832c182 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-27T07:26:41,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:41,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34408 deadline: 1732692461595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:41,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:41,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:41,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:41,612 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/7d745246ea4c4ac8a423e4fcd568301b is 50, key is test_row_0/A:col10/1732692400907/Put/seqid=0 2024-11-27T07:26:41,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742319_1495 (size=14741) 2024-11-27T07:26:41,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:41,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34408 deadline: 1732692461709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:41,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:41,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:41,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:41,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:41,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34408 deadline: 1732692461919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:41,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:41,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:41,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:42,015 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/7d745246ea4c4ac8a423e4fcd568301b 2024-11-27T07:26:42,021 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/08e8d93544c4488c88690dc880b99bc6 is 50, key is test_row_0/B:col10/1732692400907/Put/seqid=0 2024-11-27T07:26:42,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742320_1496 (size=12301) 2024-11-27T07:26:42,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:42,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34408 deadline: 1732692462229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:42,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:42,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:42,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:42,425 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/08e8d93544c4488c88690dc880b99bc6 2024-11-27T07:26:42,431 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/73481cfdb9234fc29ef65d1ff5302a7b is 50, key is test_row_0/C:col10/1732692400907/Put/seqid=0 2024-11-27T07:26:42,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742321_1497 (size=12301) 2024-11-27T07:26:42,434 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/73481cfdb9234fc29ef65d1ff5302a7b 2024-11-27T07:26:42,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/7d745246ea4c4ac8a423e4fcd568301b as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/7d745246ea4c4ac8a423e4fcd568301b 2024-11-27T07:26:42,440 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/7d745246ea4c4ac8a423e4fcd568301b, entries=200, sequenceid=285, filesize=14.4 K 2024-11-27T07:26:42,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/08e8d93544c4488c88690dc880b99bc6 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/08e8d93544c4488c88690dc880b99bc6 2024-11-27T07:26:42,443 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/08e8d93544c4488c88690dc880b99bc6, entries=150, sequenceid=285, filesize=12.0 K 2024-11-27T07:26:42,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/73481cfdb9234fc29ef65d1ff5302a7b as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/73481cfdb9234fc29ef65d1ff5302a7b 2024-11-27T07:26:42,447 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/73481cfdb9234fc29ef65d1ff5302a7b, entries=150, sequenceid=285, filesize=12.0 K 2024-11-27T07:26:42,448 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 8dcfd8a4aaa1fef77f3b9d89e832c182 in 894ms, sequenceid=285, compaction requested=true 2024-11-27T07:26:42,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8dcfd8a4aaa1fef77f3b9d89e832c182: 2024-11-27T07:26:42,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8dcfd8a4aaa1fef77f3b9d89e832c182:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T07:26:42,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:26:42,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8dcfd8a4aaa1fef77f3b9d89e832c182:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T07:26:42,448 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T07:26:42,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:26:42,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8dcfd8a4aaa1fef77f3b9d89e832c182:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T07:26:42,448 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T07:26:42,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:26:42,449 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49300 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T07:26:42,449 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 8dcfd8a4aaa1fef77f3b9d89e832c182/B is initiating minor compaction (all files) 2024-11-27T07:26:42,449 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8dcfd8a4aaa1fef77f3b9d89e832c182/B in TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:42,450 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/062e63c25ecd4051a75980bce05e0bef, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/d090e1c68bfe4db2bcf8475a092304ca, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/2b6cba7a5e0143029138eaef0ee629e1, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/08e8d93544c4488c88690dc880b99bc6] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp, totalSize=48.1 K 2024-11-27T07:26:42,450 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54130 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T07:26:42,450 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1541): 8dcfd8a4aaa1fef77f3b9d89e832c182/A is initiating minor compaction (all files) 2024-11-27T07:26:42,450 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8dcfd8a4aaa1fef77f3b9d89e832c182/A in TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:42,450 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/c192f94b8f3a4f2fa615c3c2c3ad9394, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/12c764db72b244748846b3c0cafd8e84, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/ae3e2fe889de4c4d929e0be900cd1061, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/7d745246ea4c4ac8a423e4fcd568301b] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp, totalSize=52.9 K 2024-11-27T07:26:42,450 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 062e63c25ecd4051a75980bce05e0bef, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1732692398315 2024-11-27T07:26:42,450 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting c192f94b8f3a4f2fa615c3c2c3ad9394, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1732692398315 2024-11-27T07:26:42,450 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting d090e1c68bfe4db2bcf8475a092304ca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732692398436 2024-11-27T07:26:42,450 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12c764db72b244748846b3c0cafd8e84, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732692398430 2024-11-27T07:26:42,451 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b6cba7a5e0143029138eaef0ee629e1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732692399612 2024-11-27T07:26:42,451 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae3e2fe889de4c4d929e0be900cd1061, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732692399612 2024-11-27T07:26:42,451 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 08e8d93544c4488c88690dc880b99bc6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1732692400851 2024-11-27T07:26:42,451 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d745246ea4c4ac8a423e4fcd568301b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1732692400851 2024-11-27T07:26:42,459 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8dcfd8a4aaa1fef77f3b9d89e832c182#B#compaction#420 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:26:42,459 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/868e24127f3f4e0dabee3ec033f9bb12 is 50, key is test_row_0/B:col10/1732692400907/Put/seqid=0 2024-11-27T07:26:42,461 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8dcfd8a4aaa1fef77f3b9d89e832c182#A#compaction#421 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:26:42,461 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/80188591fbda44dc8915882164519e11 is 50, key is test_row_0/A:col10/1732692400907/Put/seqid=0 2024-11-27T07:26:42,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742322_1498 (size=12983) 2024-11-27T07:26:42,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742323_1499 (size=12983) 2024-11-27T07:26:42,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 8dcfd8a4aaa1fef77f3b9d89e832c182 2024-11-27T07:26:42,746 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T07:26:42,746 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8dcfd8a4aaa1fef77f3b9d89e832c182 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-27T07:26:42,750 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/574294292e1845db8a4273e77cfbe362 is 50, key is test_row_0/A:col10/1732692402741/Put/seqid=0 2024-11-27T07:26:42,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742324_1500 (size=14741) 2024-11-27T07:26:42,754 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/574294292e1845db8a4273e77cfbe362 2024-11-27T07:26:42,760 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/a23a5c4973404237b5b96643f7c11929 is 50, key is test_row_0/B:col10/1732692402741/Put/seqid=0 2024-11-27T07:26:42,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742325_1501 (size=12301) 2024-11-27T07:26:42,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-27T07:26:42,769 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T07:26:42,769 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T07:26:42,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-27T07:26:42,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-27T07:26:42,770 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T07:26:42,771 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T07:26:42,771 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T07:26:42,877 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/868e24127f3f4e0dabee3ec033f9bb12 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/868e24127f3f4e0dabee3ec033f9bb12 2024-11-27T07:26:42,877 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/80188591fbda44dc8915882164519e11 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/80188591fbda44dc8915882164519e11 2024-11-27T07:26:42,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-27T07:26:42,881 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8dcfd8a4aaa1fef77f3b9d89e832c182/B of 8dcfd8a4aaa1fef77f3b9d89e832c182 into 868e24127f3f4e0dabee3ec033f9bb12(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:26:42,881 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8dcfd8a4aaa1fef77f3b9d89e832c182/A of 8dcfd8a4aaa1fef77f3b9d89e832c182 into 80188591fbda44dc8915882164519e11(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:26:42,881 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8dcfd8a4aaa1fef77f3b9d89e832c182: 2024-11-27T07:26:42,881 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8dcfd8a4aaa1fef77f3b9d89e832c182: 2024-11-27T07:26:42,881 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., storeName=8dcfd8a4aaa1fef77f3b9d89e832c182/A, priority=12, startTime=1732692402448; duration=0sec 2024-11-27T07:26:42,881 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., storeName=8dcfd8a4aaa1fef77f3b9d89e832c182/B, priority=12, startTime=1732692402448; duration=0sec 2024-11-27T07:26:42,881 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:26:42,881 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:26:42,881 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8dcfd8a4aaa1fef77f3b9d89e832c182:A 2024-11-27T07:26:42,881 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8dcfd8a4aaa1fef77f3b9d89e832c182:B 2024-11-27T07:26:42,881 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T07:26:42,882 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49300 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T07:26:42,882 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1541): 8dcfd8a4aaa1fef77f3b9d89e832c182/C is initiating minor compaction (all files) 2024-11-27T07:26:42,882 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8dcfd8a4aaa1fef77f3b9d89e832c182/C in TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:42,882 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/ae279d3f21934f70bbf03b2ae9dfec56, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/b8852b29f5764e5dbb50b00879434794, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/4db144ec6fb2410a86d5533307cf0cbb, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/73481cfdb9234fc29ef65d1ff5302a7b] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp, totalSize=48.1 K 2024-11-27T07:26:42,882 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae279d3f21934f70bbf03b2ae9dfec56, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1732692398315 2024-11-27T07:26:42,882 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8852b29f5764e5dbb50b00879434794, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732692398436 2024-11-27T07:26:42,882 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4db144ec6fb2410a86d5533307cf0cbb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732692399612 2024-11-27T07:26:42,883 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73481cfdb9234fc29ef65d1ff5302a7b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1732692400851 2024-11-27T07:26:42,889 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8dcfd8a4aaa1fef77f3b9d89e832c182#C#compaction#424 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:26:42,890 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/488745baa65c410ba6ce36951cf0950b is 50, key is test_row_0/C:col10/1732692400907/Put/seqid=0 2024-11-27T07:26:42,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742326_1502 (size=12983) 2024-11-27T07:26:42,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:42,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 248 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34408 deadline: 1732692462901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:42,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:42,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:42,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:42,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-27T07:26:42,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:42,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. as already flushing 2024-11-27T07:26:42,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:42,924 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:42,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:42,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:43,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:43,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 249 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34408 deadline: 1732692463019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:43,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:43,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:43,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:43,077 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-27T07:26:43,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:43,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. as already flushing 2024-11-27T07:26:43,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:43,077 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:43,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:43,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:43,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-27T07:26:43,168 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/a23a5c4973404237b5b96643f7c11929 2024-11-27T07:26:43,173 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/2b3f381d122340d190437c526629311b is 50, key is test_row_0/C:col10/1732692402741/Put/seqid=0 2024-11-27T07:26:43,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742327_1503 (size=12301) 2024-11-27T07:26:43,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-27T07:26:43,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:43,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. as already flushing 2024-11-27T07:26:43,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:43,229 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:43,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:43,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:43,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:43,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 250 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34408 deadline: 1732692463229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:43,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:43,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:43,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:43,297 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/488745baa65c410ba6ce36951cf0950b as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/488745baa65c410ba6ce36951cf0950b 2024-11-27T07:26:43,301 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8dcfd8a4aaa1fef77f3b9d89e832c182/C of 8dcfd8a4aaa1fef77f3b9d89e832c182 into 488745baa65c410ba6ce36951cf0950b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:26:43,301 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8dcfd8a4aaa1fef77f3b9d89e832c182: 2024-11-27T07:26:43,301 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., storeName=8dcfd8a4aaa1fef77f3b9d89e832c182/C, priority=12, startTime=1732692402448; duration=0sec 2024-11-27T07:26:43,301 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:26:43,301 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8dcfd8a4aaa1fef77f3b9d89e832c182:C 2024-11-27T07:26:43,382 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-27T07:26:43,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:43,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. as already flushing 2024-11-27T07:26:43,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:43,382 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:43,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:43,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:43,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-27T07:26:43,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-27T07:26:43,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:43,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. as already flushing 2024-11-27T07:26:43,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:43,535 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:43,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:43,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:43,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:43,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 251 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34408 deadline: 1732692463548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:43,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:43,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:43,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:43,577 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/2b3f381d122340d190437c526629311b 2024-11-27T07:26:43,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/574294292e1845db8a4273e77cfbe362 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/574294292e1845db8a4273e77cfbe362 2024-11-27T07:26:43,583 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/574294292e1845db8a4273e77cfbe362, entries=200, sequenceid=296, filesize=14.4 K 2024-11-27T07:26:43,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/a23a5c4973404237b5b96643f7c11929 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/a23a5c4973404237b5b96643f7c11929 2024-11-27T07:26:43,587 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/a23a5c4973404237b5b96643f7c11929, entries=150, sequenceid=296, filesize=12.0 K 2024-11-27T07:26:43,588 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/2b3f381d122340d190437c526629311b as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/2b3f381d122340d190437c526629311b 2024-11-27T07:26:43,591 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/2b3f381d122340d190437c526629311b, entries=150, sequenceid=296, filesize=12.0 K 2024-11-27T07:26:43,591 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 8dcfd8a4aaa1fef77f3b9d89e832c182 in 845ms, sequenceid=296, compaction requested=false 2024-11-27T07:26:43,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8dcfd8a4aaa1fef77f3b9d89e832c182: 2024-11-27T07:26:43,687 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-27T07:26:43,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:43,687 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 8dcfd8a4aaa1fef77f3b9d89e832c182 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-27T07:26:43,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/1d759d12acd24c9cb0515b21254d2525 is 50, key is test_row_0/A:col10/1732692402840/Put/seqid=0 2024-11-27T07:26:43,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742328_1504 (size=12301) 2024-11-27T07:26:43,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-27T07:26:44,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 8dcfd8a4aaa1fef77f3b9d89e832c182 2024-11-27T07:26:44,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. as already flushing 2024-11-27T07:26:44,095 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/1d759d12acd24c9cb0515b21254d2525 2024-11-27T07:26:44,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:44,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 260 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34408 deadline: 1732692464091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:44,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:44,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:44,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:44,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/656d488c6be842eaacfb261f2f6ad0ff is 50, key is test_row_0/B:col10/1732692402840/Put/seqid=0 2024-11-27T07:26:44,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742329_1505 (size=12301) 2024-11-27T07:26:44,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:44,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 261 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34408 deadline: 1732692464199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:44,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:44,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:44,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:44,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:44,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 262 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34408 deadline: 1732692464409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:44,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:44,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:44,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:44,504 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/656d488c6be842eaacfb261f2f6ad0ff 2024-11-27T07:26:44,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/8e88feb732bd4fe282d4cc07982ec140 is 50, key is test_row_0/C:col10/1732692402840/Put/seqid=0 2024-11-27T07:26:44,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742330_1506 (size=12301) 2024-11-27T07:26:44,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:44,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 263 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34408 deadline: 1732692464719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:44,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:44,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:44,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:44,913 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/8e88feb732bd4fe282d4cc07982ec140 2024-11-27T07:26:44,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/1d759d12acd24c9cb0515b21254d2525 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/1d759d12acd24c9cb0515b21254d2525 2024-11-27T07:26:44,918 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/1d759d12acd24c9cb0515b21254d2525, entries=150, sequenceid=324, filesize=12.0 K 2024-11-27T07:26:44,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-27T07:26:44,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/656d488c6be842eaacfb261f2f6ad0ff as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/656d488c6be842eaacfb261f2f6ad0ff 2024-11-27T07:26:44,922 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/656d488c6be842eaacfb261f2f6ad0ff, entries=150, sequenceid=324, filesize=12.0 K 2024-11-27T07:26:44,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/8e88feb732bd4fe282d4cc07982ec140 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/8e88feb732bd4fe282d4cc07982ec140 2024-11-27T07:26:44,926 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/8e88feb732bd4fe282d4cc07982ec140, entries=150, sequenceid=324, filesize=12.0 K 2024-11-27T07:26:44,926 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 8dcfd8a4aaa1fef77f3b9d89e832c182 in 1239ms, sequenceid=324, compaction requested=true 2024-11-27T07:26:44,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 8dcfd8a4aaa1fef77f3b9d89e832c182: 2024-11-27T07:26:44,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:44,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-27T07:26:44,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-27T07:26:44,929 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-27T07:26:44,929 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1560 sec 2024-11-27T07:26:44,930 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 2.1600 sec 2024-11-27T07:26:45,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 8dcfd8a4aaa1fef77f3b9d89e832c182 2024-11-27T07:26:45,244 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T07:26:45,244 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8dcfd8a4aaa1fef77f3b9d89e832c182 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-27T07:26:45,248 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/d8e958c48db44789a541dad204ca2f90 is 50, key is test_row_0/A:col10/1732692404090/Put/seqid=0 2024-11-27T07:26:45,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742331_1507 (size=14741) 2024-11-27T07:26:45,253 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/d8e958c48db44789a541dad204ca2f90 2024-11-27T07:26:45,260 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/77abbbeaae1645efad6baf9cfc4509e0 is 50, key is test_row_0/B:col10/1732692404090/Put/seqid=0 2024-11-27T07:26:45,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742332_1508 (size=12301) 2024-11-27T07:26:45,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:45,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 286 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34408 deadline: 1732692465408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:45,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:45,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:45,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:45,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:45,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 287 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34408 deadline: 1732692465519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:45,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:45,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:45,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:45,664 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/77abbbeaae1645efad6baf9cfc4509e0 2024-11-27T07:26:45,670 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/a0ef3e00c21a47bc94b01b9e6579286e is 50, key is test_row_0/C:col10/1732692404090/Put/seqid=0 2024-11-27T07:26:45,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742333_1509 (size=12301) 2024-11-27T07:26:45,700 INFO [Thread-1981 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1981. 2024-11-27T07:26:45,700 DEBUG [Thread-1981 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T07:26:45,700 DEBUG [Thread-1981 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:26:45,700 DEBUG [Thread-1981 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:26:45,700 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T07:26:45,705 INFO [Thread-1983 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1983. 2024-11-27T07:26:45,705 DEBUG [Thread-1983 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T07:26:45,705 DEBUG [Thread-1983 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:26:45,705 DEBUG [Thread-1983 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:26:45,705 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T07:26:45,705 INFO [Thread-1977 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1977. 2024-11-27T07:26:45,705 DEBUG [Thread-1977 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T07:26:45,705 DEBUG [Thread-1977 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:26:45,705 DEBUG [Thread-1977 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:26:45,705 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T07:26:45,709 INFO [Thread-1975 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1975. 2024-11-27T07:26:45,709 DEBUG [Thread-1975 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T07:26:45,709 DEBUG [Thread-1975 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:26:45,709 DEBUG [Thread-1975 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:26:45,709 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T07:26:45,713 INFO [Thread-1979 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1979. 2024-11-27T07:26:45,713 DEBUG [Thread-1979 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T07:26:45,713 DEBUG [Thread-1979 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:26:45,713 DEBUG [Thread-1979 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:26:45,713 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T07:26:45,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:45,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 288 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34408 deadline: 1732692465739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:45,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:45,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:45,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:46,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:46,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 289 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:34408 deadline: 1732692466050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:46,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 , the old value is region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:46,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:46,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., hostname=289954c9202d,42117,1732692259671, seqNum=2 because the exception is null or not the one we care about 2024-11-27T07:26:46,074 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/a0ef3e00c21a47bc94b01b9e6579286e 2024-11-27T07:26:46,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/d8e958c48db44789a541dad204ca2f90 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/d8e958c48db44789a541dad204ca2f90 2024-11-27T07:26:46,090 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/d8e958c48db44789a541dad204ca2f90, entries=200, sequenceid=336, filesize=14.4 K 2024-11-27T07:26:46,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/77abbbeaae1645efad6baf9cfc4509e0 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/77abbbeaae1645efad6baf9cfc4509e0 2024-11-27T07:26:46,094 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/77abbbeaae1645efad6baf9cfc4509e0, entries=150, sequenceid=336, filesize=12.0 K 2024-11-27T07:26:46,095 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/a0ef3e00c21a47bc94b01b9e6579286e as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/a0ef3e00c21a47bc94b01b9e6579286e 2024-11-27T07:26:46,098 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/a0ef3e00c21a47bc94b01b9e6579286e, entries=150, sequenceid=336, filesize=12.0 K 2024-11-27T07:26:46,099 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 8dcfd8a4aaa1fef77f3b9d89e832c182 in 855ms, sequenceid=336, compaction requested=true 2024-11-27T07:26:46,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8dcfd8a4aaa1fef77f3b9d89e832c182: 2024-11-27T07:26:46,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8dcfd8a4aaa1fef77f3b9d89e832c182:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T07:26:46,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:26:46,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8dcfd8a4aaa1fef77f3b9d89e832c182:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T07:26:46,099 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T07:26:46,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:26:46,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8dcfd8a4aaa1fef77f3b9d89e832c182:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T07:26:46,099 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T07:26:46,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:26:46,100 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54766 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T07:26:46,100 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T07:26:46,101 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 8dcfd8a4aaa1fef77f3b9d89e832c182/B is initiating minor compaction (all files) 2024-11-27T07:26:46,101 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1541): 8dcfd8a4aaa1fef77f3b9d89e832c182/A is initiating minor compaction (all files) 2024-11-27T07:26:46,101 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8dcfd8a4aaa1fef77f3b9d89e832c182/B in TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:46,101 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8dcfd8a4aaa1fef77f3b9d89e832c182/A in TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:46,101 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/868e24127f3f4e0dabee3ec033f9bb12, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/a23a5c4973404237b5b96643f7c11929, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/656d488c6be842eaacfb261f2f6ad0ff, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/77abbbeaae1645efad6baf9cfc4509e0] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp, totalSize=48.7 K 2024-11-27T07:26:46,101 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/80188591fbda44dc8915882164519e11, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/574294292e1845db8a4273e77cfbe362, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/1d759d12acd24c9cb0515b21254d2525, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/d8e958c48db44789a541dad204ca2f90] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp, totalSize=53.5 K 2024-11-27T07:26:46,101 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 868e24127f3f4e0dabee3ec033f9bb12, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1732692400851 2024-11-27T07:26:46,101 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80188591fbda44dc8915882164519e11, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1732692400851 2024-11-27T07:26:46,101 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting a23a5c4973404237b5b96643f7c11929, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1732692401577 2024-11-27T07:26:46,101 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 574294292e1845db8a4273e77cfbe362, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1732692401558 2024-11-27T07:26:46,102 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 656d488c6be842eaacfb261f2f6ad0ff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732692402840 2024-11-27T07:26:46,102 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d759d12acd24c9cb0515b21254d2525, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732692402840 2024-11-27T07:26:46,102 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 77abbbeaae1645efad6baf9cfc4509e0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732692404074 2024-11-27T07:26:46,102 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8e958c48db44789a541dad204ca2f90, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732692404074 2024-11-27T07:26:46,109 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8dcfd8a4aaa1fef77f3b9d89e832c182#B#compaction#433 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:26:46,110 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8dcfd8a4aaa1fef77f3b9d89e832c182#A#compaction#432 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:26:46,110 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/ac4cd61a6e8b4c58bee0ae769fb6fdc1 is 50, key is test_row_0/B:col10/1732692404090/Put/seqid=0 2024-11-27T07:26:46,110 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/bf1330b0e89e4c788d26c04acbcf4546 is 50, key is test_row_0/A:col10/1732692404090/Put/seqid=0 2024-11-27T07:26:46,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742334_1510 (size=13119) 2024-11-27T07:26:46,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742335_1511 (size=13119) 2024-11-27T07:26:46,525 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/ac4cd61a6e8b4c58bee0ae769fb6fdc1 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/ac4cd61a6e8b4c58bee0ae769fb6fdc1 2024-11-27T07:26:46,525 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/bf1330b0e89e4c788d26c04acbcf4546 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/bf1330b0e89e4c788d26c04acbcf4546 2024-11-27T07:26:46,530 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8dcfd8a4aaa1fef77f3b9d89e832c182/A of 8dcfd8a4aaa1fef77f3b9d89e832c182 into bf1330b0e89e4c788d26c04acbcf4546(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:26:46,530 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8dcfd8a4aaa1fef77f3b9d89e832c182/B of 8dcfd8a4aaa1fef77f3b9d89e832c182 into ac4cd61a6e8b4c58bee0ae769fb6fdc1(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:26:46,530 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8dcfd8a4aaa1fef77f3b9d89e832c182: 2024-11-27T07:26:46,530 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8dcfd8a4aaa1fef77f3b9d89e832c182: 2024-11-27T07:26:46,530 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., storeName=8dcfd8a4aaa1fef77f3b9d89e832c182/B, priority=12, startTime=1732692406099; duration=0sec 2024-11-27T07:26:46,530 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., storeName=8dcfd8a4aaa1fef77f3b9d89e832c182/A, priority=12, startTime=1732692406099; duration=0sec 2024-11-27T07:26:46,530 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:26:46,530 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:26:46,530 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8dcfd8a4aaa1fef77f3b9d89e832c182:B 2024-11-27T07:26:46,530 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8dcfd8a4aaa1fef77f3b9d89e832c182:A 2024-11-27T07:26:46,530 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T07:26:46,531 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T07:26:46,532 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 8dcfd8a4aaa1fef77f3b9d89e832c182/C is initiating minor compaction (all files) 2024-11-27T07:26:46,532 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8dcfd8a4aaa1fef77f3b9d89e832c182/C in TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:46,532 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/488745baa65c410ba6ce36951cf0950b, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/2b3f381d122340d190437c526629311b, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/8e88feb732bd4fe282d4cc07982ec140, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/a0ef3e00c21a47bc94b01b9e6579286e] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp, totalSize=48.7 K 2024-11-27T07:26:46,532 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 488745baa65c410ba6ce36951cf0950b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1732692400851 2024-11-27T07:26:46,532 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b3f381d122340d190437c526629311b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1732692401577 2024-11-27T07:26:46,532 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e88feb732bd4fe282d4cc07982ec140, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732692402840 2024-11-27T07:26:46,533 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting a0ef3e00c21a47bc94b01b9e6579286e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732692404074 2024-11-27T07:26:46,540 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8dcfd8a4aaa1fef77f3b9d89e832c182#C#compaction#434 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:26:46,540 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/080b868a9a2949d8b50dfcb0413bfe5b is 50, key is test_row_0/C:col10/1732692404090/Put/seqid=0 2024-11-27T07:26:46,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742336_1512 (size=13119) 2024-11-27T07:26:46,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 8dcfd8a4aaa1fef77f3b9d89e832c182 2024-11-27T07:26:46,620 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T07:26:46,620 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8dcfd8a4aaa1fef77f3b9d89e832c182 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-27T07:26:46,620 INFO [Thread-1970 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1970. 2024-11-27T07:26:46,621 DEBUG [Thread-1970 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T07:26:46,621 DEBUG [Thread-1970 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:26:46,621 DEBUG [Thread-1970 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:26:46,621 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T07:26:46,624 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/5f036ba6b17c45598e7ebba04a6fa76e is 50, key is test_row_0/A:col10/1732692405383/Put/seqid=0 2024-11-27T07:26:46,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742337_1513 (size=12301) 2024-11-27T07:26:46,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-27T07:26:46,929 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T07:26:46,951 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/080b868a9a2949d8b50dfcb0413bfe5b as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/080b868a9a2949d8b50dfcb0413bfe5b 2024-11-27T07:26:46,960 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8dcfd8a4aaa1fef77f3b9d89e832c182/C of 8dcfd8a4aaa1fef77f3b9d89e832c182 into 080b868a9a2949d8b50dfcb0413bfe5b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:26:46,960 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8dcfd8a4aaa1fef77f3b9d89e832c182: 2024-11-27T07:26:46,960 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., storeName=8dcfd8a4aaa1fef77f3b9d89e832c182/C, priority=12, startTime=1732692406099; duration=0sec 2024-11-27T07:26:46,960 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:26:46,960 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8dcfd8a4aaa1fef77f3b9d89e832c182:C 2024-11-27T07:26:47,028 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/5f036ba6b17c45598e7ebba04a6fa76e 2024-11-27T07:26:47,036 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/6a4455b3417d42bb9c6ba233dd61d2a8 is 50, key is test_row_0/B:col10/1732692405383/Put/seqid=0 2024-11-27T07:26:47,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742338_1514 (size=12301) 2024-11-27T07:26:47,050 INFO [Thread-1968 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1968. 2024-11-27T07:26:47,050 DEBUG [Thread-1968 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T07:26:47,050 DEBUG [Thread-1968 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:26:47,050 DEBUG [Thread-1968 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:26:47,050 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T07:26:47,072 INFO [Thread-1964 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1964. 2024-11-27T07:26:47,072 INFO [Thread-1972 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1972. 2024-11-27T07:26:47,072 DEBUG [Thread-1964 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T07:26:47,072 DEBUG [Thread-1972 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T07:26:47,072 DEBUG [Thread-1964 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:26:47,072 DEBUG [Thread-1972 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:26:47,073 DEBUG [Thread-1964 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:26:47,073 DEBUG [Thread-1972 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:26:47,073 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T07:26:47,073 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T07:26:47,103 INFO [Thread-1966 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1966. 2024-11-27T07:26:47,103 DEBUG [Thread-1966 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T07:26:47,103 DEBUG [Thread-1966 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:26:47,103 DEBUG [Thread-1966 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:26:47,103 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T07:26:47,104 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-27T07:26:47,104 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 16 2024-11-27T07:26:47,104 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 16 2024-11-27T07:26:47,104 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 17 2024-11-27T07:26:47,104 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 226 2024-11-27T07:26:47,104 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 14 2024-11-27T07:26:47,104 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-27T07:26:47,104 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-27T07:26:47,105 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1588 2024-11-27T07:26:47,105 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4761 rows 2024-11-27T07:26:47,105 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1584 2024-11-27T07:26:47,105 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4750 rows 2024-11-27T07:26:47,105 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1578 2024-11-27T07:26:47,105 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4733 rows 2024-11-27T07:26:47,105 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1571 2024-11-27T07:26:47,105 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4710 rows 2024-11-27T07:26:47,105 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1582 2024-11-27T07:26:47,105 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4746 rows 2024-11-27T07:26:47,105 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T07:26:47,105 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T07:26:47,106 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:26:47,106 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:26:47,106 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T07:26:47,108 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-27T07:26:47,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-27T07:26:47,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-27T07:26:47,113 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732692407112"}]},"ts":"1732692407112"} 2024-11-27T07:26:47,114 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-27T07:26:47,114 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-27T07:26:47,115 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-27T07:26:47,117 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8dcfd8a4aaa1fef77f3b9d89e832c182, UNASSIGN}] 2024-11-27T07:26:47,118 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8dcfd8a4aaa1fef77f3b9d89e832c182, UNASSIGN 2024-11-27T07:26:47,119 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=8dcfd8a4aaa1fef77f3b9d89e832c182, regionState=CLOSING, regionLocation=289954c9202d,42117,1732692259671 2024-11-27T07:26:47,121 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8dcfd8a4aaa1fef77f3b9d89e832c182, UNASSIGN because future has completed 2024-11-27T07:26:47,121 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T07:26:47,121 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671}] 2024-11-27T07:26:47,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-27T07:26:47,275 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(122): Close 8dcfd8a4aaa1fef77f3b9d89e832c182 2024-11-27T07:26:47,276 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T07:26:47,276 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1722): Closing 8dcfd8a4aaa1fef77f3b9d89e832c182, disabling compactions & flushes 2024-11-27T07:26:47,277 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:47,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-27T07:26:47,442 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/6a4455b3417d42bb9c6ba233dd61d2a8 2024-11-27T07:26:47,457 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/606615ed70054bf7b792038b23df247a is 50, key is test_row_0/C:col10/1732692405383/Put/seqid=0 2024-11-27T07:26:47,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742339_1515 (size=12301) 2024-11-27T07:26:47,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-27T07:26:47,862 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/606615ed70054bf7b792038b23df247a 2024-11-27T07:26:47,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/5f036ba6b17c45598e7ebba04a6fa76e as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/5f036ba6b17c45598e7ebba04a6fa76e 2024-11-27T07:26:47,876 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/5f036ba6b17c45598e7ebba04a6fa76e, entries=150, sequenceid=363, filesize=12.0 K 2024-11-27T07:26:47,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/6a4455b3417d42bb9c6ba233dd61d2a8 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/6a4455b3417d42bb9c6ba233dd61d2a8 2024-11-27T07:26:47,880 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/6a4455b3417d42bb9c6ba233dd61d2a8, entries=150, sequenceid=363, filesize=12.0 K 2024-11-27T07:26:47,881 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/606615ed70054bf7b792038b23df247a as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/606615ed70054bf7b792038b23df247a 2024-11-27T07:26:47,884 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/606615ed70054bf7b792038b23df247a, entries=150, sequenceid=363, filesize=12.0 K 2024-11-27T07:26:47,885 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=26.84 KB/27480 for 8dcfd8a4aaa1fef77f3b9d89e832c182 in 1265ms, sequenceid=363, compaction requested=false 2024-11-27T07:26:47,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8dcfd8a4aaa1fef77f3b9d89e832c182: 2024-11-27T07:26:47,885 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:47,885 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:47,885 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. after waiting 0 ms 2024-11-27T07:26:47,885 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:47,885 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(2902): Flushing 8dcfd8a4aaa1fef77f3b9d89e832c182 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-27T07:26:47,887 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/5a3cc51cde344995aac66b5dc0298582 is 50, key is test_row_0/A:col10/1732692407100/Put/seqid=0 2024-11-27T07:26:47,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742340_1516 (size=12301) 2024-11-27T07:26:47,970 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-27T07:26:48,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-27T07:26:48,293 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/5a3cc51cde344995aac66b5dc0298582 2024-11-27T07:26:48,306 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/bedb2da726204886802cf6e5d2aef9a3 is 50, key is test_row_0/B:col10/1732692407100/Put/seqid=0 2024-11-27T07:26:48,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742341_1517 (size=12301) 2024-11-27T07:26:48,712 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/bedb2da726204886802cf6e5d2aef9a3 2024-11-27T07:26:48,724 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/b35b404d90bd42199741f90050371d43 is 50, key is test_row_0/C:col10/1732692407100/Put/seqid=0 2024-11-27T07:26:48,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742342_1518 (size=12301) 2024-11-27T07:26:49,129 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/b35b404d90bd42199741f90050371d43 2024-11-27T07:26:49,139 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/A/5a3cc51cde344995aac66b5dc0298582 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/5a3cc51cde344995aac66b5dc0298582 2024-11-27T07:26:49,143 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/5a3cc51cde344995aac66b5dc0298582, entries=150, sequenceid=371, filesize=12.0 K 2024-11-27T07:26:49,144 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/B/bedb2da726204886802cf6e5d2aef9a3 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/bedb2da726204886802cf6e5d2aef9a3 2024-11-27T07:26:49,148 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/bedb2da726204886802cf6e5d2aef9a3, entries=150, sequenceid=371, filesize=12.0 K 2024-11-27T07:26:49,149 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/.tmp/C/b35b404d90bd42199741f90050371d43 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/b35b404d90bd42199741f90050371d43 2024-11-27T07:26:49,153 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/b35b404d90bd42199741f90050371d43, entries=150, sequenceid=371, filesize=12.0 K 2024-11-27T07:26:49,154 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 8dcfd8a4aaa1fef77f3b9d89e832c182 in 1269ms, sequenceid=371, compaction requested=true 2024-11-27T07:26:49,155 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/21462d73ce4c42bd985073e454240f3f, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/d4392dd3e2694f0e9e7faebcf9a12278, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/a469ef39b2164542aad8b5f8fea486c0, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/699e81b22f574df590cbebfb0fe86886, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/2fc98998b96547bbbf3a33964abf3496, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/b12187803f0042758e34f7dfea64ebdb, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/fdde9133dd894ba9890b2bf717587236, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/8e650a82de704b11a376ea9959bef178, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/d48cd8c45ad0456d841d30d54196295e, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/480ddac70a75493182bad3c88e248e4d, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/05076e579605448085ce31a3f1010132, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/3fedb318eda44227bc030169856bd112, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/82390dac06e94ccf802eef8c9acd77c1, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/3f6022809d6f4a0fb54b1fbec3c0dc5b, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/f391d6167fc04954a08a3e1e8f6c82cf, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/16f21f9201d4431a9e93111cacf861ea, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/c192f94b8f3a4f2fa615c3c2c3ad9394, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/12c764db72b244748846b3c0cafd8e84, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/ae3e2fe889de4c4d929e0be900cd1061, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/7d745246ea4c4ac8a423e4fcd568301b, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/80188591fbda44dc8915882164519e11, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/574294292e1845db8a4273e77cfbe362, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/1d759d12acd24c9cb0515b21254d2525, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/d8e958c48db44789a541dad204ca2f90] to archive 2024-11-27T07:26:49,156 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T07:26:49,158 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/21462d73ce4c42bd985073e454240f3f to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/21462d73ce4c42bd985073e454240f3f 2024-11-27T07:26:49,160 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/d4392dd3e2694f0e9e7faebcf9a12278 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/d4392dd3e2694f0e9e7faebcf9a12278 2024-11-27T07:26:49,161 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/a469ef39b2164542aad8b5f8fea486c0 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/a469ef39b2164542aad8b5f8fea486c0 2024-11-27T07:26:49,162 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/699e81b22f574df590cbebfb0fe86886 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/699e81b22f574df590cbebfb0fe86886 2024-11-27T07:26:49,163 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/2fc98998b96547bbbf3a33964abf3496 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/2fc98998b96547bbbf3a33964abf3496 2024-11-27T07:26:49,163 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/b12187803f0042758e34f7dfea64ebdb to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/b12187803f0042758e34f7dfea64ebdb 2024-11-27T07:26:49,164 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/fdde9133dd894ba9890b2bf717587236 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/fdde9133dd894ba9890b2bf717587236 2024-11-27T07:26:49,165 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/8e650a82de704b11a376ea9959bef178 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/8e650a82de704b11a376ea9959bef178 2024-11-27T07:26:49,166 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/d48cd8c45ad0456d841d30d54196295e to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/d48cd8c45ad0456d841d30d54196295e 2024-11-27T07:26:49,166 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/480ddac70a75493182bad3c88e248e4d to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/480ddac70a75493182bad3c88e248e4d 2024-11-27T07:26:49,167 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/05076e579605448085ce31a3f1010132 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/05076e579605448085ce31a3f1010132 2024-11-27T07:26:49,168 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/3fedb318eda44227bc030169856bd112 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/3fedb318eda44227bc030169856bd112 2024-11-27T07:26:49,169 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/82390dac06e94ccf802eef8c9acd77c1 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/82390dac06e94ccf802eef8c9acd77c1 2024-11-27T07:26:49,170 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/3f6022809d6f4a0fb54b1fbec3c0dc5b to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/3f6022809d6f4a0fb54b1fbec3c0dc5b 2024-11-27T07:26:49,170 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/f391d6167fc04954a08a3e1e8f6c82cf to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/f391d6167fc04954a08a3e1e8f6c82cf 2024-11-27T07:26:49,171 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/16f21f9201d4431a9e93111cacf861ea to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/16f21f9201d4431a9e93111cacf861ea 2024-11-27T07:26:49,172 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/c192f94b8f3a4f2fa615c3c2c3ad9394 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/c192f94b8f3a4f2fa615c3c2c3ad9394 2024-11-27T07:26:49,172 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/12c764db72b244748846b3c0cafd8e84 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/12c764db72b244748846b3c0cafd8e84 2024-11-27T07:26:49,173 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/ae3e2fe889de4c4d929e0be900cd1061 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/ae3e2fe889de4c4d929e0be900cd1061 2024-11-27T07:26:49,174 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/7d745246ea4c4ac8a423e4fcd568301b to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/7d745246ea4c4ac8a423e4fcd568301b 2024-11-27T07:26:49,174 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/80188591fbda44dc8915882164519e11 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/80188591fbda44dc8915882164519e11 2024-11-27T07:26:49,175 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/574294292e1845db8a4273e77cfbe362 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/574294292e1845db8a4273e77cfbe362 2024-11-27T07:26:49,175 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/1d759d12acd24c9cb0515b21254d2525 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/1d759d12acd24c9cb0515b21254d2525 2024-11-27T07:26:49,176 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/d8e958c48db44789a541dad204ca2f90 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/d8e958c48db44789a541dad204ca2f90 2024-11-27T07:26:49,177 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/88beedf7496040838601d8ce45671f31, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/4719df548a2d4ea9a8c3e257db5a8672, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/f843736d5a9a41faa471db9eb4eea69d, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/03d3c65033a14472a4a36797b39af996, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/cf46b45862e34036b478f1cd2e22ff95, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/5975724ed97d42668491574bd8390804, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/111f52b9ef6345399a6b1df2e00f2d56, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/c5d15ff1a9b84ce0a4194c3ecfb88685, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/5483fafb56e44f9ba9318c857d30453c, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/fa69831e760c4516a9ae353194157605, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/56a631a4ed72405a88b83ccebaf38acc, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/b4368286e31949ea90f49b244938b68f, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/6a69ad383a5947cc9c8a6e7f7d3b63c5, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/4fed3a9b99814a36a620fbb231f25776, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/061a51660d3c464391186767d83f3f2f, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/062e63c25ecd4051a75980bce05e0bef, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/b7cbfdfade8b46938e8bced49133ae05, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/d090e1c68bfe4db2bcf8475a092304ca, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/2b6cba7a5e0143029138eaef0ee629e1, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/868e24127f3f4e0dabee3ec033f9bb12, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/08e8d93544c4488c88690dc880b99bc6, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/a23a5c4973404237b5b96643f7c11929, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/656d488c6be842eaacfb261f2f6ad0ff, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/77abbbeaae1645efad6baf9cfc4509e0] to archive 2024-11-27T07:26:49,178 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T07:26:49,179 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/88beedf7496040838601d8ce45671f31 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/88beedf7496040838601d8ce45671f31 2024-11-27T07:26:49,180 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/4719df548a2d4ea9a8c3e257db5a8672 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/4719df548a2d4ea9a8c3e257db5a8672 2024-11-27T07:26:49,180 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/f843736d5a9a41faa471db9eb4eea69d to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/f843736d5a9a41faa471db9eb4eea69d 2024-11-27T07:26:49,181 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/03d3c65033a14472a4a36797b39af996 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/03d3c65033a14472a4a36797b39af996 2024-11-27T07:26:49,182 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/cf46b45862e34036b478f1cd2e22ff95 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/cf46b45862e34036b478f1cd2e22ff95 2024-11-27T07:26:49,183 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/5975724ed97d42668491574bd8390804 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/5975724ed97d42668491574bd8390804 2024-11-27T07:26:49,183 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/111f52b9ef6345399a6b1df2e00f2d56 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/111f52b9ef6345399a6b1df2e00f2d56 2024-11-27T07:26:49,184 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/c5d15ff1a9b84ce0a4194c3ecfb88685 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/c5d15ff1a9b84ce0a4194c3ecfb88685 2024-11-27T07:26:49,185 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/5483fafb56e44f9ba9318c857d30453c to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/5483fafb56e44f9ba9318c857d30453c 2024-11-27T07:26:49,185 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/fa69831e760c4516a9ae353194157605 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/fa69831e760c4516a9ae353194157605 2024-11-27T07:26:49,186 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/56a631a4ed72405a88b83ccebaf38acc to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/56a631a4ed72405a88b83ccebaf38acc 2024-11-27T07:26:49,187 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/b4368286e31949ea90f49b244938b68f to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/b4368286e31949ea90f49b244938b68f 2024-11-27T07:26:49,187 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/6a69ad383a5947cc9c8a6e7f7d3b63c5 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/6a69ad383a5947cc9c8a6e7f7d3b63c5 2024-11-27T07:26:49,188 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/4fed3a9b99814a36a620fbb231f25776 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/4fed3a9b99814a36a620fbb231f25776 2024-11-27T07:26:49,189 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/061a51660d3c464391186767d83f3f2f to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/061a51660d3c464391186767d83f3f2f 2024-11-27T07:26:49,189 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/062e63c25ecd4051a75980bce05e0bef to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/062e63c25ecd4051a75980bce05e0bef 2024-11-27T07:26:49,190 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/b7cbfdfade8b46938e8bced49133ae05 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/b7cbfdfade8b46938e8bced49133ae05 2024-11-27T07:26:49,191 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/d090e1c68bfe4db2bcf8475a092304ca to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/d090e1c68bfe4db2bcf8475a092304ca 2024-11-27T07:26:49,192 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/2b6cba7a5e0143029138eaef0ee629e1 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/2b6cba7a5e0143029138eaef0ee629e1 2024-11-27T07:26:49,192 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/868e24127f3f4e0dabee3ec033f9bb12 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/868e24127f3f4e0dabee3ec033f9bb12 2024-11-27T07:26:49,193 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/08e8d93544c4488c88690dc880b99bc6 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/08e8d93544c4488c88690dc880b99bc6 2024-11-27T07:26:49,194 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/a23a5c4973404237b5b96643f7c11929 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/a23a5c4973404237b5b96643f7c11929 2024-11-27T07:26:49,194 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/656d488c6be842eaacfb261f2f6ad0ff to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/656d488c6be842eaacfb261f2f6ad0ff 2024-11-27T07:26:49,195 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/77abbbeaae1645efad6baf9cfc4509e0 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/77abbbeaae1645efad6baf9cfc4509e0 2024-11-27T07:26:49,196 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/d79c221309a94958a6d5207638ae8653, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/d48748f08cb4494eaeba821c76ded266, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/92b99c42f0654efaa80cbff77dd5ea2c, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/4bca193f361e4334a830f52cb235f638, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/b40df3e04b034d16a9d718028fd1daaa, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/accfbc5af49d45d6bb7724020d0635be, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/017b5341b71b4609b3633db75d567263, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/317f7703b9384fa5ac762d8ae87325fa, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/5e90acddd7a54870b0be5cfb2db79dd7, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/ba4dba4ef3ce47929c6f3868d96fe051, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/83024e0ed2694c7182946d4112b1595b, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/db9e40a6b86f46e09863674589cae253, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/5ef85c01816f4cdd9b05c45d0eae7d86, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/b97998b68a564d45bf825a28ba090cef, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/bd04e02b74d94f1abf5994b363da24c0, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/ae279d3f21934f70bbf03b2ae9dfec56, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/5ea27d631f5645b98fbcb87b0261bc2f, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/b8852b29f5764e5dbb50b00879434794, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/4db144ec6fb2410a86d5533307cf0cbb, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/488745baa65c410ba6ce36951cf0950b, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/73481cfdb9234fc29ef65d1ff5302a7b, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/2b3f381d122340d190437c526629311b, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/8e88feb732bd4fe282d4cc07982ec140, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/a0ef3e00c21a47bc94b01b9e6579286e] to archive 2024-11-27T07:26:49,197 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T07:26:49,198 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/d79c221309a94958a6d5207638ae8653 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/d79c221309a94958a6d5207638ae8653 2024-11-27T07:26:49,199 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/d48748f08cb4494eaeba821c76ded266 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/d48748f08cb4494eaeba821c76ded266 2024-11-27T07:26:49,199 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/92b99c42f0654efaa80cbff77dd5ea2c to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/92b99c42f0654efaa80cbff77dd5ea2c 2024-11-27T07:26:49,200 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/4bca193f361e4334a830f52cb235f638 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/4bca193f361e4334a830f52cb235f638 2024-11-27T07:26:49,201 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/b40df3e04b034d16a9d718028fd1daaa to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/b40df3e04b034d16a9d718028fd1daaa 2024-11-27T07:26:49,201 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/accfbc5af49d45d6bb7724020d0635be to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/accfbc5af49d45d6bb7724020d0635be 2024-11-27T07:26:49,202 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/017b5341b71b4609b3633db75d567263 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/017b5341b71b4609b3633db75d567263 2024-11-27T07:26:49,202 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/317f7703b9384fa5ac762d8ae87325fa to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/317f7703b9384fa5ac762d8ae87325fa 2024-11-27T07:26:49,203 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/5e90acddd7a54870b0be5cfb2db79dd7 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/5e90acddd7a54870b0be5cfb2db79dd7 2024-11-27T07:26:49,203 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/ba4dba4ef3ce47929c6f3868d96fe051 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/ba4dba4ef3ce47929c6f3868d96fe051 2024-11-27T07:26:49,204 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/83024e0ed2694c7182946d4112b1595b to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/83024e0ed2694c7182946d4112b1595b 2024-11-27T07:26:49,205 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/db9e40a6b86f46e09863674589cae253 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/db9e40a6b86f46e09863674589cae253 2024-11-27T07:26:49,205 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/5ef85c01816f4cdd9b05c45d0eae7d86 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/5ef85c01816f4cdd9b05c45d0eae7d86 2024-11-27T07:26:49,206 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/b97998b68a564d45bf825a28ba090cef to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/b97998b68a564d45bf825a28ba090cef 2024-11-27T07:26:49,206 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/bd04e02b74d94f1abf5994b363da24c0 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/bd04e02b74d94f1abf5994b363da24c0 2024-11-27T07:26:49,207 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/ae279d3f21934f70bbf03b2ae9dfec56 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/ae279d3f21934f70bbf03b2ae9dfec56 2024-11-27T07:26:49,208 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/5ea27d631f5645b98fbcb87b0261bc2f to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/5ea27d631f5645b98fbcb87b0261bc2f 2024-11-27T07:26:49,208 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/b8852b29f5764e5dbb50b00879434794 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/b8852b29f5764e5dbb50b00879434794 2024-11-27T07:26:49,209 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/4db144ec6fb2410a86d5533307cf0cbb to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/4db144ec6fb2410a86d5533307cf0cbb 2024-11-27T07:26:49,210 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/488745baa65c410ba6ce36951cf0950b to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/488745baa65c410ba6ce36951cf0950b 2024-11-27T07:26:49,210 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/73481cfdb9234fc29ef65d1ff5302a7b to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/73481cfdb9234fc29ef65d1ff5302a7b 2024-11-27T07:26:49,211 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/2b3f381d122340d190437c526629311b to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/2b3f381d122340d190437c526629311b 2024-11-27T07:26:49,211 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/8e88feb732bd4fe282d4cc07982ec140 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/8e88feb732bd4fe282d4cc07982ec140 2024-11-27T07:26:49,212 DEBUG [StoreCloser-TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/a0ef3e00c21a47bc94b01b9e6579286e to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/a0ef3e00c21a47bc94b01b9e6579286e 2024-11-27T07:26:49,215 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/recovered.edits/374.seqid, newMaxSeqId=374, maxSeqId=1 2024-11-27T07:26:49,216 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. 2024-11-27T07:26:49,216 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1676): Region close journal for 8dcfd8a4aaa1fef77f3b9d89e832c182: Waiting for close lock at 1732692407276Running coprocessor pre-close hooks at 1732692407276Disabling compacts and flushes for region at 1732692407276Disabling writes for close at 1732692407885 (+609 ms)Obtaining lock to block concurrent updates at 1732692407885Preparing flush snapshotting stores in 8dcfd8a4aaa1fef77f3b9d89e832c182 at 1732692407885Finished memstore snapshotting TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1732692407885Flushing stores of TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182. at 1732692407885Flushing 8dcfd8a4aaa1fef77f3b9d89e832c182/A: creating writer at 1732692407885Flushing 8dcfd8a4aaa1fef77f3b9d89e832c182/A: appending metadata at 1732692407887 (+2 ms)Flushing 8dcfd8a4aaa1fef77f3b9d89e832c182/A: closing flushed file at 1732692407887Flushing 8dcfd8a4aaa1fef77f3b9d89e832c182/B: creating writer at 1732692408303 (+416 ms)Flushing 8dcfd8a4aaa1fef77f3b9d89e832c182/B: appending metadata at 1732692408305 (+2 ms)Flushing 8dcfd8a4aaa1fef77f3b9d89e832c182/B: closing flushed file at 1732692408305Flushing 8dcfd8a4aaa1fef77f3b9d89e832c182/C: creating writer at 1732692408720 (+415 ms)Flushing 8dcfd8a4aaa1fef77f3b9d89e832c182/C: appending metadata at 1732692408723 (+3 ms)Flushing 8dcfd8a4aaa1fef77f3b9d89e832c182/C: closing flushed file at 1732692408723Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@67335240: reopening flushed file at 1732692409138 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@53fe8e60: reopening flushed file at 1732692409143 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@179cf1d0: reopening flushed file at 1732692409148 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 8dcfd8a4aaa1fef77f3b9d89e832c182 in 1269ms, sequenceid=371, compaction requested=true at 1732692409154 (+6 ms)Writing region close event to WAL at 1732692409213 (+59 ms)Running coprocessor post-close hooks at 1732692409216 (+3 ms)Closed at 1732692409216 2024-11-27T07:26:49,218 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(157): Closed 8dcfd8a4aaa1fef77f3b9d89e832c182 2024-11-27T07:26:49,218 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=8dcfd8a4aaa1fef77f3b9d89e832c182, regionState=CLOSED 2024-11-27T07:26:49,219 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 because future has completed 2024-11-27T07:26:49,221 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-27T07:26:49,221 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; CloseRegionProcedure 8dcfd8a4aaa1fef77f3b9d89e832c182, server=289954c9202d,42117,1732692259671 in 2.0990 sec 2024-11-27T07:26:49,222 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-11-27T07:26:49,222 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8dcfd8a4aaa1fef77f3b9d89e832c182, UNASSIGN in 2.1040 sec 2024-11-27T07:26:49,224 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-27T07:26:49,224 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.1080 sec 2024-11-27T07:26:49,225 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732692409225"}]},"ts":"1732692409225"} 2024-11-27T07:26:49,226 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-27T07:26:49,226 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-27T07:26:49,228 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.1190 sec 2024-11-27T07:26:49,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-27T07:26:49,260 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-27T07:26:49,261 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-27T07:26:49,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T07:26:49,264 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=141, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T07:26:49,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-27T07:26:49,265 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=141, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T07:26:49,268 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182 2024-11-27T07:26:49,270 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-27T07:26:49,272 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A, FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B, FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C, FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/recovered.edits] 2024-11-27T07:26:49,274 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/5a3cc51cde344995aac66b5dc0298582 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/5a3cc51cde344995aac66b5dc0298582 2024-11-27T07:26:49,276 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/5f036ba6b17c45598e7ebba04a6fa76e to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/5f036ba6b17c45598e7ebba04a6fa76e 2024-11-27T07:26:49,277 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/bf1330b0e89e4c788d26c04acbcf4546 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/A/bf1330b0e89e4c788d26c04acbcf4546 2024-11-27T07:26:49,279 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/6a4455b3417d42bb9c6ba233dd61d2a8 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/6a4455b3417d42bb9c6ba233dd61d2a8 2024-11-27T07:26:49,280 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/ac4cd61a6e8b4c58bee0ae769fb6fdc1 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/ac4cd61a6e8b4c58bee0ae769fb6fdc1 2024-11-27T07:26:49,282 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/bedb2da726204886802cf6e5d2aef9a3 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/B/bedb2da726204886802cf6e5d2aef9a3 2024-11-27T07:26:49,284 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/080b868a9a2949d8b50dfcb0413bfe5b to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/080b868a9a2949d8b50dfcb0413bfe5b 2024-11-27T07:26:49,285 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/606615ed70054bf7b792038b23df247a to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/606615ed70054bf7b792038b23df247a 2024-11-27T07:26:49,286 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/b35b404d90bd42199741f90050371d43 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/C/b35b404d90bd42199741f90050371d43 2024-11-27T07:26:49,288 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/recovered.edits/374.seqid to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182/recovered.edits/374.seqid 2024-11-27T07:26:49,288 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/8dcfd8a4aaa1fef77f3b9d89e832c182 2024-11-27T07:26:49,288 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-27T07:26:49,290 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=141, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T07:26:49,292 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-27T07:26:49,293 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-27T07:26:49,294 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=141, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T07:26:49,294 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-27T07:26:49,294 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732692409294"}]},"ts":"9223372036854775807"} 2024-11-27T07:26:49,296 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-27T07:26:49,296 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 8dcfd8a4aaa1fef77f3b9d89e832c182, NAME => 'TestAcidGuarantees,,1732692383494.8dcfd8a4aaa1fef77f3b9d89e832c182.', STARTKEY => '', ENDKEY => ''}] 2024-11-27T07:26:49,296 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-27T07:26:49,296 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732692409296"}]},"ts":"9223372036854775807"} 2024-11-27T07:26:49,297 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-27T07:26:49,298 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=141, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T07:26:49,299 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 37 msec 2024-11-27T07:26:49,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-27T07:26:49,369 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-27T07:26:49,372 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-27T07:26:49,381 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=240 (was 239) - Thread LEAK? -, OpenFileDescriptor=449 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=446 (was 506), ProcessCount=11 (was 11), AvailableMemoryMB=4474 (was 4567) 2024-11-27T07:26:49,391 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=240, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=446, ProcessCount=11, AvailableMemoryMB=4474 2024-11-27T07:26:49,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-27T07:26:49,393 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T07:26:49,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] procedure2.ProcedureExecutor(1139): Stored pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-27T07:26:49,395 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T07:26:49,395 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:26:49,395 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 142 2024-11-27T07:26:49,396 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T07:26:49,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-27T07:26:49,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742343_1519 (size=959) 2024-11-27T07:26:49,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-27T07:26:49,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-27T07:26:49,804 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4 2024-11-27T07:26:49,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742344_1520 (size=53) 2024-11-27T07:26:50,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-27T07:26:50,214 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T07:26:50,214 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 866f4f82d2f771c80496e5b80211b690, disabling compactions & flushes 2024-11-27T07:26:50,214 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:26:50,214 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:26:50,214 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. after waiting 0 ms 2024-11-27T07:26:50,214 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:26:50,214 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:26:50,215 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 866f4f82d2f771c80496e5b80211b690: Waiting for close lock at 1732692410214Disabling compacts and flushes for region at 1732692410214Disabling writes for close at 1732692410214Writing region close event to WAL at 1732692410214Closed at 1732692410214 2024-11-27T07:26:50,218 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T07:26:50,218 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732692410218"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732692410218"}]},"ts":"1732692410218"} 2024-11-27T07:26:50,221 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T07:26:50,222 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T07:26:50,222 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732692410222"}]},"ts":"1732692410222"} 2024-11-27T07:26:50,224 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-27T07:26:50,224 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=866f4f82d2f771c80496e5b80211b690, ASSIGN}] 2024-11-27T07:26:50,225 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=866f4f82d2f771c80496e5b80211b690, ASSIGN 2024-11-27T07:26:50,226 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=866f4f82d2f771c80496e5b80211b690, ASSIGN; state=OFFLINE, location=289954c9202d,42117,1732692259671; forceNewPlan=false, retain=false 2024-11-27T07:26:50,377 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=866f4f82d2f771c80496e5b80211b690, regionState=OPENING, regionLocation=289954c9202d,42117,1732692259671 2024-11-27T07:26:50,379 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=866f4f82d2f771c80496e5b80211b690, ASSIGN because future has completed 2024-11-27T07:26:50,379 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure 866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671}] 2024-11-27T07:26:50,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-27T07:26:50,540 INFO [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:26:50,540 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7752): Opening region: {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} 2024-11-27T07:26:50,541 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 866f4f82d2f771c80496e5b80211b690 2024-11-27T07:26:50,541 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T07:26:50,541 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7794): checking encryption for 866f4f82d2f771c80496e5b80211b690 2024-11-27T07:26:50,541 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7797): checking classloading for 866f4f82d2f771c80496e5b80211b690 2024-11-27T07:26:50,543 INFO [StoreOpener-866f4f82d2f771c80496e5b80211b690-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 866f4f82d2f771c80496e5b80211b690 2024-11-27T07:26:50,545 INFO [StoreOpener-866f4f82d2f771c80496e5b80211b690-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 866f4f82d2f771c80496e5b80211b690 columnFamilyName A 2024-11-27T07:26:50,545 DEBUG [StoreOpener-866f4f82d2f771c80496e5b80211b690-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:26:50,545 INFO [StoreOpener-866f4f82d2f771c80496e5b80211b690-1 {}] regionserver.HStore(327): Store=866f4f82d2f771c80496e5b80211b690/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T07:26:50,545 INFO [StoreOpener-866f4f82d2f771c80496e5b80211b690-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 866f4f82d2f771c80496e5b80211b690 2024-11-27T07:26:50,547 INFO [StoreOpener-866f4f82d2f771c80496e5b80211b690-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 866f4f82d2f771c80496e5b80211b690 columnFamilyName B 2024-11-27T07:26:50,547 DEBUG [StoreOpener-866f4f82d2f771c80496e5b80211b690-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:26:50,547 INFO [StoreOpener-866f4f82d2f771c80496e5b80211b690-1 {}] regionserver.HStore(327): Store=866f4f82d2f771c80496e5b80211b690/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T07:26:50,547 INFO [StoreOpener-866f4f82d2f771c80496e5b80211b690-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 866f4f82d2f771c80496e5b80211b690 2024-11-27T07:26:50,549 INFO [StoreOpener-866f4f82d2f771c80496e5b80211b690-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 866f4f82d2f771c80496e5b80211b690 columnFamilyName C 2024-11-27T07:26:50,549 DEBUG [StoreOpener-866f4f82d2f771c80496e5b80211b690-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:26:50,549 INFO [StoreOpener-866f4f82d2f771c80496e5b80211b690-1 {}] regionserver.HStore(327): Store=866f4f82d2f771c80496e5b80211b690/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T07:26:50,549 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1038): replaying wal for 866f4f82d2f771c80496e5b80211b690 2024-11-27T07:26:50,550 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690 2024-11-27T07:26:50,550 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690 2024-11-27T07:26:50,552 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1048): stopping wal replay for 866f4f82d2f771c80496e5b80211b690 2024-11-27T07:26:50,552 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1060): Cleaning up temporary data for 866f4f82d2f771c80496e5b80211b690 2024-11-27T07:26:50,552 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T07:26:50,554 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1093): writing seq id for 866f4f82d2f771c80496e5b80211b690 2024-11-27T07:26:50,556 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T07:26:50,557 INFO [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1114): Opened 866f4f82d2f771c80496e5b80211b690; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65699201, jitterRate=-0.02100561559200287}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T07:26:50,557 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 866f4f82d2f771c80496e5b80211b690 2024-11-27T07:26:50,558 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1006): Region open journal for 866f4f82d2f771c80496e5b80211b690: Running coprocessor pre-open hook at 1732692410541Writing region info on filesystem at 1732692410541Initializing all the Stores at 1732692410542 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732692410542Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732692410542Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732692410543 (+1 ms)Cleaning up temporary data from old regions at 1732692410552 (+9 ms)Running coprocessor post-open hooks at 1732692410557 (+5 ms)Region opened successfully at 1732692410558 (+1 ms) 2024-11-27T07:26:50,559 INFO [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., pid=144, masterSystemTime=1732692410532 2024-11-27T07:26:50,561 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:26:50,561 INFO [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:26:50,562 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=866f4f82d2f771c80496e5b80211b690, regionState=OPEN, openSeqNum=2, regionLocation=289954c9202d,42117,1732692259671 2024-11-27T07:26:50,564 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure 866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 because future has completed 2024-11-27T07:26:50,567 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-27T07:26:50,567 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; OpenRegionProcedure 866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 in 186 msec 2024-11-27T07:26:50,569 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-27T07:26:50,569 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=866f4f82d2f771c80496e5b80211b690, ASSIGN in 343 msec 2024-11-27T07:26:50,570 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T07:26:50,570 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732692410570"}]},"ts":"1732692410570"} 2024-11-27T07:26:50,572 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-27T07:26:50,573 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T07:26:50,575 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1800 sec 2024-11-27T07:26:51,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-27T07:26:51,550 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-27T07:26:51,552 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1832522d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:26:51,552 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 289954c9202d,43057,-1 for getting cluster id 2024-11-27T07:26:51,553 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T07:26:51,554 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '09f43d3f-c175-4713-a9ce-038f5695b74b' 2024-11-27T07:26:51,555 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T07:26:51,555 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "09f43d3f-c175-4713-a9ce-038f5695b74b" 2024-11-27T07:26:51,556 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cbec9c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:26:51,556 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [289954c9202d,43057,-1] 2024-11-27T07:26:51,556 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T07:26:51,557 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:26:51,558 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44390, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T07:26:51,559 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2739f3f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:26:51,560 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T07:26:51,560 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=289954c9202d,42117,1732692259671, seqNum=-1] 2024-11-27T07:26:51,561 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T07:26:51,561 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39132, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T07:26:51,562 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-27T07:26:51,563 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 289954c9202d,43057,1732692258941 2024-11-27T07:26:51,563 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1647b0e1 2024-11-27T07:26:51,563 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-27T07:26:51,563 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44404, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-27T07:26:51,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-27T07:26:51,565 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T07:26:51,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-27T07:26:51,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742345_1521 (size=995) 2024-11-27T07:26:51,661 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-27T07:26:51,661 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-27T07:26:51,664 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-27T07:26:51,667 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=866f4f82d2f771c80496e5b80211b690, REOPEN/MOVE}] 2024-11-27T07:26:51,668 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=866f4f82d2f771c80496e5b80211b690, REOPEN/MOVE 2024-11-27T07:26:51,669 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=866f4f82d2f771c80496e5b80211b690, regionState=CLOSING, regionLocation=289954c9202d,42117,1732692259671 2024-11-27T07:26:51,671 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=866f4f82d2f771c80496e5b80211b690, REOPEN/MOVE because future has completed 2024-11-27T07:26:51,671 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T07:26:51,671 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure 866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671}] 2024-11-27T07:26:51,853 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(122): Close 866f4f82d2f771c80496e5b80211b690 2024-11-27T07:26:51,853 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T07:26:51,854 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1722): Closing 866f4f82d2f771c80496e5b80211b690, disabling compactions & flushes 2024-11-27T07:26:51,854 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:26:51,854 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:26:51,854 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. after waiting 0 ms 2024-11-27T07:26:51,854 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:26:51,860 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-27T07:26:51,861 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:26:51,861 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1676): Region close journal for 866f4f82d2f771c80496e5b80211b690: Waiting for close lock at 1732692411853Running coprocessor pre-close hooks at 1732692411853Disabling compacts and flushes for region at 1732692411853Disabling writes for close at 1732692411854 (+1 ms)Writing region close event to WAL at 1732692411855 (+1 ms)Running coprocessor post-close hooks at 1732692411860 (+5 ms)Closed at 1732692411861 (+1 ms) 2024-11-27T07:26:51,861 WARN [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegionServer(3299): Not adding moved region record: 866f4f82d2f771c80496e5b80211b690 to self. 2024-11-27T07:26:51,863 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(157): Closed 866f4f82d2f771c80496e5b80211b690 2024-11-27T07:26:51,863 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=866f4f82d2f771c80496e5b80211b690, regionState=CLOSED 2024-11-27T07:26:51,865 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure 866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 because future has completed 2024-11-27T07:26:51,868 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-27T07:26:51,868 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; CloseRegionProcedure 866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 in 195 msec 2024-11-27T07:26:51,869 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=866f4f82d2f771c80496e5b80211b690, REOPEN/MOVE; state=CLOSED, location=289954c9202d,42117,1732692259671; forceNewPlan=false, retain=true 2024-11-27T07:26:52,020 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=866f4f82d2f771c80496e5b80211b690, regionState=OPENING, regionLocation=289954c9202d,42117,1732692259671 2024-11-27T07:26:52,022 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=866f4f82d2f771c80496e5b80211b690, REOPEN/MOVE because future has completed 2024-11-27T07:26:52,022 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure 866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671}] 2024-11-27T07:26:52,182 INFO [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:26:52,182 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7752): Opening region: {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} 2024-11-27T07:26:52,183 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 866f4f82d2f771c80496e5b80211b690 2024-11-27T07:26:52,183 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T07:26:52,183 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7794): checking encryption for 866f4f82d2f771c80496e5b80211b690 2024-11-27T07:26:52,183 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7797): checking classloading for 866f4f82d2f771c80496e5b80211b690 2024-11-27T07:26:52,185 INFO [StoreOpener-866f4f82d2f771c80496e5b80211b690-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 866f4f82d2f771c80496e5b80211b690 2024-11-27T07:26:52,186 INFO [StoreOpener-866f4f82d2f771c80496e5b80211b690-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 866f4f82d2f771c80496e5b80211b690 columnFamilyName A 2024-11-27T07:26:52,187 DEBUG [StoreOpener-866f4f82d2f771c80496e5b80211b690-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:26:52,188 INFO [StoreOpener-866f4f82d2f771c80496e5b80211b690-1 {}] regionserver.HStore(327): Store=866f4f82d2f771c80496e5b80211b690/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T07:26:52,188 INFO [StoreOpener-866f4f82d2f771c80496e5b80211b690-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 866f4f82d2f771c80496e5b80211b690 2024-11-27T07:26:52,189 INFO [StoreOpener-866f4f82d2f771c80496e5b80211b690-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 866f4f82d2f771c80496e5b80211b690 columnFamilyName B 2024-11-27T07:26:52,189 DEBUG [StoreOpener-866f4f82d2f771c80496e5b80211b690-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:26:52,189 INFO [StoreOpener-866f4f82d2f771c80496e5b80211b690-1 {}] regionserver.HStore(327): Store=866f4f82d2f771c80496e5b80211b690/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T07:26:52,189 INFO [StoreOpener-866f4f82d2f771c80496e5b80211b690-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 866f4f82d2f771c80496e5b80211b690 2024-11-27T07:26:52,190 INFO [StoreOpener-866f4f82d2f771c80496e5b80211b690-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 866f4f82d2f771c80496e5b80211b690 columnFamilyName C 2024-11-27T07:26:52,190 DEBUG [StoreOpener-866f4f82d2f771c80496e5b80211b690-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:26:52,190 INFO [StoreOpener-866f4f82d2f771c80496e5b80211b690-1 {}] regionserver.HStore(327): Store=866f4f82d2f771c80496e5b80211b690/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T07:26:52,191 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1038): replaying wal for 866f4f82d2f771c80496e5b80211b690 2024-11-27T07:26:52,191 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690 2024-11-27T07:26:52,192 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690 2024-11-27T07:26:52,193 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1048): stopping wal replay for 866f4f82d2f771c80496e5b80211b690 2024-11-27T07:26:52,193 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1060): Cleaning up temporary data for 866f4f82d2f771c80496e5b80211b690 2024-11-27T07:26:52,194 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T07:26:52,195 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1093): writing seq id for 866f4f82d2f771c80496e5b80211b690 2024-11-27T07:26:52,195 INFO [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1114): Opened 866f4f82d2f771c80496e5b80211b690; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75092445, jitterRate=0.11896462738513947}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T07:26:52,195 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 866f4f82d2f771c80496e5b80211b690 2024-11-27T07:26:52,196 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1006): Region open journal for 866f4f82d2f771c80496e5b80211b690: Running coprocessor pre-open hook at 1732692412184Writing region info on filesystem at 1732692412184Initializing all the Stores at 1732692412185 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732692412185Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732692412185Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732692412185Cleaning up temporary data from old regions at 1732692412193 (+8 ms)Running coprocessor post-open hooks at 1732692412195 (+2 ms)Region opened successfully at 1732692412196 (+1 ms) 2024-11-27T07:26:52,197 INFO [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., pid=149, masterSystemTime=1732692412174 2024-11-27T07:26:52,198 DEBUG [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:26:52,198 INFO [RS_OPEN_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:26:52,198 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=866f4f82d2f771c80496e5b80211b690, regionState=OPEN, openSeqNum=5, regionLocation=289954c9202d,42117,1732692259671 2024-11-27T07:26:52,199 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure 866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 because future has completed 2024-11-27T07:26:52,201 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=147 2024-11-27T07:26:52,201 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=147, state=SUCCESS, hasLock=false; OpenRegionProcedure 866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 in 178 msec 2024-11-27T07:26:52,203 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-27T07:26:52,203 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=866f4f82d2f771c80496e5b80211b690, REOPEN/MOVE in 534 msec 2024-11-27T07:26:52,204 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-27T07:26:52,204 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 539 msec 2024-11-27T07:26:52,206 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 640 msec 2024-11-27T07:26:52,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-27T07:26:52,207 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-27T07:26:52,207 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22faf5ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:26:52,207 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 289954c9202d,43057,-1 for getting cluster id 2024-11-27T07:26:52,208 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T07:26:52,208 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '09f43d3f-c175-4713-a9ce-038f5695b74b' 2024-11-27T07:26:52,208 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T07:26:52,208 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "09f43d3f-c175-4713-a9ce-038f5695b74b" 2024-11-27T07:26:52,208 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f3892c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:26:52,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [289954c9202d,43057,-1] 2024-11-27T07:26:52,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T07:26:52,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:26:52,209 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41706, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T07:26:52,210 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71eaf571, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:26:52,210 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@744b0f71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:26:52,210 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 289954c9202d,43057,-1 for getting cluster id 2024-11-27T07:26:52,210 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T07:26:52,211 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '09f43d3f-c175-4713-a9ce-038f5695b74b' 2024-11-27T07:26:52,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T07:26:52,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "09f43d3f-c175-4713-a9ce-038f5695b74b" 2024-11-27T07:26:52,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e9b22f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:26:52,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [289954c9202d,43057,-1] 2024-11-27T07:26:52,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T07:26:52,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:26:52,212 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41722, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T07:26:52,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7407cbe9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:26:52,213 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@143fb08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:26:52,213 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 289954c9202d,43057,-1 for getting cluster id 2024-11-27T07:26:52,213 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T07:26:52,213 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '09f43d3f-c175-4713-a9ce-038f5695b74b' 2024-11-27T07:26:52,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T07:26:52,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "09f43d3f-c175-4713-a9ce-038f5695b74b" 2024-11-27T07:26:52,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130841f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:26:52,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [289954c9202d,43057,-1] 2024-11-27T07:26:52,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T07:26:52,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:26:52,214 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41744, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T07:26:52,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3144925c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:26:52,216 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40503a36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:26:52,216 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 289954c9202d,43057,-1 for getting cluster id 2024-11-27T07:26:52,216 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T07:26:52,216 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '09f43d3f-c175-4713-a9ce-038f5695b74b' 2024-11-27T07:26:52,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T07:26:52,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "09f43d3f-c175-4713-a9ce-038f5695b74b" 2024-11-27T07:26:52,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32947683, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:26:52,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [289954c9202d,43057,-1] 2024-11-27T07:26:52,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T07:26:52,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:26:52,217 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41768, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T07:26:52,218 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a2462c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:26:52,218 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52f72e1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:26:52,218 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 289954c9202d,43057,-1 for getting cluster id 2024-11-27T07:26:52,218 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T07:26:52,219 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '09f43d3f-c175-4713-a9ce-038f5695b74b' 2024-11-27T07:26:52,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T07:26:52,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "09f43d3f-c175-4713-a9ce-038f5695b74b" 2024-11-27T07:26:52,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f704304, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:26:52,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [289954c9202d,43057,-1] 2024-11-27T07:26:52,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T07:26:52,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:26:52,220 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41794, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T07:26:52,220 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a7a752b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:26:52,221 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@763ca8cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:26:52,221 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 289954c9202d,43057,-1 for getting cluster id 2024-11-27T07:26:52,221 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T07:26:52,222 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '09f43d3f-c175-4713-a9ce-038f5695b74b' 2024-11-27T07:26:52,222 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T07:26:52,222 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "09f43d3f-c175-4713-a9ce-038f5695b74b" 2024-11-27T07:26:52,222 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fcf56a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:26:52,222 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [289954c9202d,43057,-1] 2024-11-27T07:26:52,222 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T07:26:52,222 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:26:52,223 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41816, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T07:26:52,223 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2de8812b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:26:52,223 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7122db2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:26:52,223 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 289954c9202d,43057,-1 for getting cluster id 2024-11-27T07:26:52,223 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T07:26:52,224 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '09f43d3f-c175-4713-a9ce-038f5695b74b' 2024-11-27T07:26:52,224 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T07:26:52,224 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "09f43d3f-c175-4713-a9ce-038f5695b74b" 2024-11-27T07:26:52,224 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4babc00b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:26:52,224 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [289954c9202d,43057,-1] 2024-11-27T07:26:52,224 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T07:26:52,225 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:26:52,225 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41840, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T07:26:52,226 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d4f3822, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:26:52,226 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3666b173, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:26:52,226 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 289954c9202d,43057,-1 for getting cluster id 2024-11-27T07:26:52,226 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T07:26:52,227 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '09f43d3f-c175-4713-a9ce-038f5695b74b' 2024-11-27T07:26:52,227 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T07:26:52,227 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "09f43d3f-c175-4713-a9ce-038f5695b74b" 2024-11-27T07:26:52,227 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62db7ae1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:26:52,227 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [289954c9202d,43057,-1] 2024-11-27T07:26:52,227 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T07:26:52,228 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:26:52,228 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41858, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T07:26:52,229 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28cb727a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:26:52,229 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77ff027c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:26:52,229 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 289954c9202d,43057,-1 for getting cluster id 2024-11-27T07:26:52,229 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T07:26:52,230 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '09f43d3f-c175-4713-a9ce-038f5695b74b' 2024-11-27T07:26:52,230 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T07:26:52,230 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "09f43d3f-c175-4713-a9ce-038f5695b74b" 2024-11-27T07:26:52,230 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d93314e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:26:52,230 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [289954c9202d,43057,-1] 2024-11-27T07:26:52,230 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T07:26:52,231 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:26:52,231 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41878, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T07:26:52,232 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3350fd3f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:26:52,232 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2637386d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:26:52,232 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 289954c9202d,43057,-1 for getting cluster id 2024-11-27T07:26:52,232 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T07:26:52,233 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '09f43d3f-c175-4713-a9ce-038f5695b74b' 2024-11-27T07:26:52,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T07:26:52,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "09f43d3f-c175-4713-a9ce-038f5695b74b" 2024-11-27T07:26:52,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73af5498, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:26:52,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [289954c9202d,43057,-1] 2024-11-27T07:26:52,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T07:26:52,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:26:52,234 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41902, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T07:26:52,235 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c98122e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T07:26:52,235 DEBUG [Thread-2347 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T07:26:52,235 DEBUG [Thread-2330 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T07:26:52,236 DEBUG [Thread-2345 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T07:26:52,236 DEBUG [Thread-2334 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T07:26:52,236 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T07:26:52,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=289954c9202d,42117,1732692259671, seqNum=-1] 2024-11-27T07:26:52,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T07:26:52,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=289954c9202d,42117,1732692259671, seqNum=-1] 2024-11-27T07:26:52,236 DEBUG [Thread-2339 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T07:26:52,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T07:26:52,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=289954c9202d,42117,1732692259671, seqNum=-1] 2024-11-27T07:26:52,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=289954c9202d,42117,1732692259671, seqNum=-1] 2024-11-27T07:26:52,237 DEBUG [Thread-2336 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T07:26:52,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T07:26:52,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T07:26:52,237 DEBUG [Thread-2328 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T07:26:52,237 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59794, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T07:26:52,237 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=289954c9202d,42117,1732692259671, seqNum=-1] 2024-11-27T07:26:52,238 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T07:26:52,238 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59828, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T07:26:52,238 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59808, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T07:26:52,238 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59810, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T07:26:52,238 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=289954c9202d,42117,1732692259671, seqNum=-1] 2024-11-27T07:26:52,238 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T07:26:52,239 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59854, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T07:26:52,239 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=289954c9202d,42117,1732692259671, seqNum=-1] 2024-11-27T07:26:52,239 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T07:26:52,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5] 2024-11-27T07:26:52,240 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59860, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T07:26:52,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5] 2024-11-27T07:26:52,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5] 2024-11-27T07:26:52,243 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5] 2024-11-27T07:26:52,243 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59840, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T07:26:52,243 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5] 2024-11-27T07:26:52,244 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5] 2024-11-27T07:26:52,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-27T07:26:52,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-27T07:26:52,245 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T07:26:52,246 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T07:26:52,246 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T07:26:52,247 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5] 2024-11-27T07:26:52,248 DEBUG [Thread-2341 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T07:26:52,248 DEBUG [Thread-2343 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T07:26:52,249 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=289954c9202d,42117,1732692259671, seqNum=-1] 2024-11-27T07:26:52,249 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=289954c9202d,42117,1732692259671, seqNum=-1] 2024-11-27T07:26:52,249 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T07:26:52,249 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T07:26:52,249 DEBUG [Thread-2332 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T07:26:52,250 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59872, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T07:26:52,250 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59874, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T07:26:52,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=289954c9202d,42117,1732692259671, seqNum=-1] 2024-11-27T07:26:52,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T07:26:52,251 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59888, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T07:26:52,252 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5] 2024-11-27T07:26:52,252 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5] 2024-11-27T07:26:52,253 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5] 2024-11-27T07:26:52,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 866f4f82d2f771c80496e5b80211b690 2024-11-27T07:26:52,253 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T07:26:52,254 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 866f4f82d2f771c80496e5b80211b690 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-27T07:26:52,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:52,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:52,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59810 deadline: 1732692472273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:52,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59860 deadline: 1732692472273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:52,278 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:52,278 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:52,278 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:52,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:52,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:52,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1732692472275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:52,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59888 deadline: 1732692472274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:52,278 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112752a672fba7b54d0ab52695fc679e37cd_866f4f82d2f771c80496e5b80211b690 is 50, key is test_row_0/A:col10/1732692412250/Put/seqid=0 2024-11-27T07:26:52,278 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:52,278 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:52,278 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:52,279 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:52,279 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:52,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:52,279 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:52,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:52,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:52,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:52,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1732692472278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:52,281 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:52,281 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:52,281 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:52,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742346_1522 (size=14594) 2024-11-27T07:26:52,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-27T07:26:52,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:52,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59810 deadline: 1732692472389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:52,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:52,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:52,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:52,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:52,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59860 deadline: 1732692472389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:52,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:52,391 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:52,391 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:52,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59888 deadline: 1732692472389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:52,391 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:52,392 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:52,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:52,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:52,392 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:52,392 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:52,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1732692472389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:52,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1732692472390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:52,392 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:52,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:52,392 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:52,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:52,392 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:52,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:52,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-27T07:26:52,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:26:52,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. as already flushing 2024-11-27T07:26:52,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:26:52,402 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:52,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:52,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:52,553 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-27T07:26:52,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:26:52,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. as already flushing 2024-11-27T07:26:52,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:26:52,554 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:52,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:52,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:52,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-27T07:26:52,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:52,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59810 deadline: 1732692472599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:52,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:52,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:52,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59860 deadline: 1732692472599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:52,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:52,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:52,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:52,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59888 deadline: 1732692472599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:52,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:52,602 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:52,602 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:52,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1732692472600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:52,602 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:52,602 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:52,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:52,602 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:52,602 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:52,602 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:52,602 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:52,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1732692472600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:52,602 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:52,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:52,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:52,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:52,690 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:26:52,692 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112752a672fba7b54d0ab52695fc679e37cd_866f4f82d2f771c80496e5b80211b690 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112752a672fba7b54d0ab52695fc679e37cd_866f4f82d2f771c80496e5b80211b690 2024-11-27T07:26:52,693 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/A/9477368a26614e42bf26c794bf89a48e, store: [table=TestAcidGuarantees family=A region=866f4f82d2f771c80496e5b80211b690] 2024-11-27T07:26:52,694 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/A/9477368a26614e42bf26c794bf89a48e is 175, key is test_row_0/A:col10/1732692412250/Put/seqid=0 2024-11-27T07:26:52,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742347_1523 (size=39549) 2024-11-27T07:26:52,697 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/A/9477368a26614e42bf26c794bf89a48e 2024-11-27T07:26:52,706 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-27T07:26:52,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:26:52,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. as already flushing 2024-11-27T07:26:52,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:26:52,706 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:52,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:52,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:52,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/B/47c724e907e14a61865d440ef649fdd1 is 50, key is test_row_0/B:col10/1732692412250/Put/seqid=0 2024-11-27T07:26:52,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742348_1524 (size=12001) 2024-11-27T07:26:52,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-27T07:26:52,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:26:52,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. as already flushing 2024-11-27T07:26:52,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:26:52,859 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:52,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:52,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:52,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-27T07:26:52,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:52,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59810 deadline: 1732692472909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:52,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:52,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59860 deadline: 1732692472909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:52,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:52,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:52,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:52,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59888 deadline: 1732692472909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:52,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:52,912 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:52,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:52,912 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:52,912 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:52,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1732692472909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:52,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:52,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1732692472909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:52,912 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:52,912 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:52,912 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:52,912 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:52,912 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:52,912 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:52,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:52,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:52,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:53,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-27T07:26:53,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:26:53,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. as already flushing 2024-11-27T07:26:53,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:26:53,012 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:53,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:53,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:53,121 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/B/47c724e907e14a61865d440ef649fdd1 2024-11-27T07:26:53,138 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/C/90063e32a3444686b89317add3fabd8e is 50, key is test_row_0/C:col10/1732692412250/Put/seqid=0 2024-11-27T07:26:53,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742349_1525 (size=12001) 2024-11-27T07:26:53,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-27T07:26:53,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:26:53,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. as already flushing 2024-11-27T07:26:53,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:26:53,164 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:53,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:53,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:53,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-27T07:26:53,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:26:53,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. as already flushing 2024-11-27T07:26:53,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:26:53,317 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:53,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:53,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:53,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-27T07:26:53,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:53,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59810 deadline: 1732692473419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:53,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:53,421 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:53,421 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:53,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1732692473419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:53,421 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:53,421 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:53,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:53,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:53,421 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:53,421 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:53,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1732692473419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:53,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59860 deadline: 1732692473419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:53,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:53,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59888 deadline: 1732692473419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:53,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:53,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:53,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:53,422 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:53,422 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:53,422 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:53,422 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:53,422 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:53,422 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:53,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-27T07:26:53,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:26:53,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. as already flushing 2024-11-27T07:26:53,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:26:53,469 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:53,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:53,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:53,541 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/C/90063e32a3444686b89317add3fabd8e 2024-11-27T07:26:53,544 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/A/9477368a26614e42bf26c794bf89a48e as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/9477368a26614e42bf26c794bf89a48e 2024-11-27T07:26:53,550 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/9477368a26614e42bf26c794bf89a48e, entries=200, sequenceid=16, filesize=38.6 K 2024-11-27T07:26:53,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/B/47c724e907e14a61865d440ef649fdd1 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/47c724e907e14a61865d440ef649fdd1 2024-11-27T07:26:53,553 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/47c724e907e14a61865d440ef649fdd1, entries=150, sequenceid=16, filesize=11.7 K 2024-11-27T07:26:53,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/C/90063e32a3444686b89317add3fabd8e as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/90063e32a3444686b89317add3fabd8e 2024-11-27T07:26:53,557 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/90063e32a3444686b89317add3fabd8e, entries=150, sequenceid=16, filesize=11.7 K 2024-11-27T07:26:53,557 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 866f4f82d2f771c80496e5b80211b690 in 1304ms, sequenceid=16, compaction requested=false 2024-11-27T07:26:53,557 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-27T07:26:53,558 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 866f4f82d2f771c80496e5b80211b690: 2024-11-27T07:26:53,622 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-27T07:26:53,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:26:53,622 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing 866f4f82d2f771c80496e5b80211b690 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-27T07:26:53,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112736a8806fa6074db3ab700809c2dc743b_866f4f82d2f771c80496e5b80211b690 is 50, key is test_row_0/A:col10/1732692412274/Put/seqid=0 2024-11-27T07:26:53,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742350_1526 (size=12154) 2024-11-27T07:26:54,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:26:54,046 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112736a8806fa6074db3ab700809c2dc743b_866f4f82d2f771c80496e5b80211b690 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112736a8806fa6074db3ab700809c2dc743b_866f4f82d2f771c80496e5b80211b690 2024-11-27T07:26:54,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/A/8e76bc29d89d4a09acbc4f2b271d3699, store: [table=TestAcidGuarantees family=A region=866f4f82d2f771c80496e5b80211b690] 2024-11-27T07:26:54,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/A/8e76bc29d89d4a09acbc4f2b271d3699 is 175, key is test_row_0/A:col10/1732692412274/Put/seqid=0 2024-11-27T07:26:54,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742351_1527 (size=30955) 2024-11-27T07:26:54,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-27T07:26:54,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. as already flushing 2024-11-27T07:26:54,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 866f4f82d2f771c80496e5b80211b690 2024-11-27T07:26:54,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:54,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59810 deadline: 1732692474437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:54,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:54,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:54,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:54,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:54,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1732692474438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:54,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:54,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1732692474438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:54,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:54,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:54,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:54,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:54,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59860 deadline: 1732692474438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:54,441 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 289954c9202d,42117,1732692259671 for 'test_row_1' in 866f4f82d2f771c80496e5b80211b690 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T07:26:54,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:54,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:54,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:54,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:54,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59888 deadline: 1732692474439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:54,442 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 289954c9202d,42117,1732692259671 for 'test_row_0' in 866f4f82d2f771c80496e5b80211b690 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T07:26:54,442 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:54,442 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:54,442 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:54,442 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:54,443 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:54,443 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:54,450 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/A/8e76bc29d89d4a09acbc4f2b271d3699 2024-11-27T07:26:54,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/B/62b5a08980c544bfa2b98228395dbaaa is 50, key is test_row_0/B:col10/1732692412274/Put/seqid=0 2024-11-27T07:26:54,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742352_1528 (size=12001) 2024-11-27T07:26:54,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:54,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59810 deadline: 1732692474548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:54,551 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:54,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:54,552 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:54,552 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:54,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1732692474548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:54,552 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:54,552 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:54,552 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:54,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:54,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59888 deadline: 1732692474549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:54,552 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:54,552 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:54,552 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:54,737 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T07:26:54,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:54,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59810 deadline: 1732692474759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:54,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:54,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:54,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:54,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:54,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59888 deadline: 1732692474759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:54,761 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:54,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:54,761 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:54,761 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:54,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1732692474759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:54,762 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:54,762 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:54,762 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:54,859 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/B/62b5a08980c544bfa2b98228395dbaaa 2024-11-27T07:26:54,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/C/399218b9e6e34ceca97f948d02a64294 is 50, key is test_row_0/C:col10/1732692412274/Put/seqid=0 2024-11-27T07:26:54,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742353_1529 (size=12001) 2024-11-27T07:26:54,868 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/C/399218b9e6e34ceca97f948d02a64294 2024-11-27T07:26:54,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/A/8e76bc29d89d4a09acbc4f2b271d3699 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/8e76bc29d89d4a09acbc4f2b271d3699 2024-11-27T07:26:54,879 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/8e76bc29d89d4a09acbc4f2b271d3699, entries=150, sequenceid=40, filesize=30.2 K 2024-11-27T07:26:54,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/B/62b5a08980c544bfa2b98228395dbaaa as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/62b5a08980c544bfa2b98228395dbaaa 2024-11-27T07:26:54,882 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/62b5a08980c544bfa2b98228395dbaaa, entries=150, sequenceid=40, filesize=11.7 K 2024-11-27T07:26:54,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/C/399218b9e6e34ceca97f948d02a64294 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/399218b9e6e34ceca97f948d02a64294 2024-11-27T07:26:54,885 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/399218b9e6e34ceca97f948d02a64294, entries=150, sequenceid=40, filesize=11.7 K 2024-11-27T07:26:54,886 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 866f4f82d2f771c80496e5b80211b690 in 1264ms, sequenceid=40, compaction requested=false 2024-11-27T07:26:54,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for 866f4f82d2f771c80496e5b80211b690: 2024-11-27T07:26:54,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:26:54,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-27T07:26:54,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-27T07:26:54,888 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-27T07:26:54,888 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6410 sec 2024-11-27T07:26:54,890 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 2.6530 sec 2024-11-27T07:26:55,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 866f4f82d2f771c80496e5b80211b690 2024-11-27T07:26:55,071 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T07:26:55,071 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 866f4f82d2f771c80496e5b80211b690 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-27T07:26:55,077 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127624982cd006641c2a181106fde3ef266_866f4f82d2f771c80496e5b80211b690 is 50, key is test_row_0/A:col10/1732692415071/Put/seqid=0 2024-11-27T07:26:55,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742354_1530 (size=14594) 2024-11-27T07:26:55,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:55,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1732692475101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:55,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:55,105 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:55,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59888 deadline: 1732692475101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:55,105 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:55,105 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:55,105 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:55,105 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:55,105 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:55,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:55,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59810 deadline: 1732692475103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:55,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:55,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:55,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:55,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:55,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1732692475209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:55,211 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:55,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:55,211 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:55,211 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:55,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59888 deadline: 1732692475209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:55,211 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:55,212 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:55,212 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:55,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:55,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59810 deadline: 1732692475209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:55,212 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:55,212 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:55,212 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:55,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:55,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1732692475419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:55,420 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:55,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:55,420 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:55,420 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:55,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59810 deadline: 1732692475419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:55,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:55,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:55,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59888 deadline: 1732692475419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:55,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:55,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:55,420 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:55,420 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:55,420 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:55,490 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:26:55,493 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127624982cd006641c2a181106fde3ef266_866f4f82d2f771c80496e5b80211b690 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127624982cd006641c2a181106fde3ef266_866f4f82d2f771c80496e5b80211b690 2024-11-27T07:26:55,494 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/A/de09385b04c94ac89d8307ed809d5079, store: [table=TestAcidGuarantees family=A region=866f4f82d2f771c80496e5b80211b690] 2024-11-27T07:26:55,494 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/A/de09385b04c94ac89d8307ed809d5079 is 175, key is test_row_0/A:col10/1732692415071/Put/seqid=0 2024-11-27T07:26:55,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742355_1531 (size=39549) 2024-11-27T07:26:55,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:55,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1732692475729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:55,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:55,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:55,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:55,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:55,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59888 deadline: 1732692475729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:55,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:55,731 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:55,731 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:55,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59810 deadline: 1732692475729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:55,731 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:55,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:55,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:55,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:55,898 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=55, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/A/de09385b04c94ac89d8307ed809d5079 2024-11-27T07:26:55,903 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/B/6dc3b53b69aa45c6bc6f5099b3cfd915 is 50, key is test_row_0/B:col10/1732692415071/Put/seqid=0 2024-11-27T07:26:55,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742356_1532 (size=12001) 2024-11-27T07:26:56,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:56,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1732692476239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:56,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:56,240 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:56,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59888 deadline: 1732692476239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:56,241 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:56,241 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:56,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:56,241 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:56,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59810 deadline: 1732692476239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:56,241 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:56,241 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:56,241 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:56,241 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:56,241 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:56,307 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/B/6dc3b53b69aa45c6bc6f5099b3cfd915 2024-11-27T07:26:56,332 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/C/89d1b340afdf4b5fa26093f52d2f109b is 50, key is test_row_0/C:col10/1732692415071/Put/seqid=0 2024-11-27T07:26:56,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742357_1533 (size=12001) 2024-11-27T07:26:56,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-27T07:26:56,409 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T07:26:56,410 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T07:26:56,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-27T07:26:56,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-27T07:26:56,411 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T07:26:56,412 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T07:26:56,412 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T07:26:56,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:56,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1732692476459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:56,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:56,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59860 deadline: 1732692476459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:56,460 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 289954c9202d,42117,1732692259671 for 'test_row_1' in 866f4f82d2f771c80496e5b80211b690 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T07:26:56,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:56,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:56,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:56,461 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 289954c9202d,42117,1732692259671 for 'test_row_0' in 866f4f82d2f771c80496e5b80211b690 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T07:26:56,461 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:56,461 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:56,461 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:56,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-27T07:26:56,564 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-27T07:26:56,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:26:56,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. as already flushing 2024-11-27T07:26:56,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:26:56,564 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:56,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:56,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:56,716 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-27T07:26:56,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:26:56,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. as already flushing 2024-11-27T07:26:56,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:26:56,717 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:56,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:56,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:56,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-27T07:26:56,735 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/C/89d1b340afdf4b5fa26093f52d2f109b 2024-11-27T07:26:56,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/A/de09385b04c94ac89d8307ed809d5079 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/de09385b04c94ac89d8307ed809d5079 2024-11-27T07:26:56,742 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/de09385b04c94ac89d8307ed809d5079, entries=200, sequenceid=55, filesize=38.6 K 2024-11-27T07:26:56,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/B/6dc3b53b69aa45c6bc6f5099b3cfd915 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/6dc3b53b69aa45c6bc6f5099b3cfd915 2024-11-27T07:26:56,745 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/6dc3b53b69aa45c6bc6f5099b3cfd915, entries=150, sequenceid=55, filesize=11.7 K 2024-11-27T07:26:56,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/C/89d1b340afdf4b5fa26093f52d2f109b as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/89d1b340afdf4b5fa26093f52d2f109b 2024-11-27T07:26:56,749 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/89d1b340afdf4b5fa26093f52d2f109b, entries=150, sequenceid=55, filesize=11.7 K 2024-11-27T07:26:56,750 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 866f4f82d2f771c80496e5b80211b690 in 1679ms, sequenceid=55, compaction requested=true 2024-11-27T07:26:56,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 866f4f82d2f771c80496e5b80211b690: 2024-11-27T07:26:56,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 866f4f82d2f771c80496e5b80211b690:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T07:26:56,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:26:56,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 866f4f82d2f771c80496e5b80211b690:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T07:26:56,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:26:56,750 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:26:56,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 866f4f82d2f771c80496e5b80211b690:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T07:26:56,750 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:26:56,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:26:56,751 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110053 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:26:56,751 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:26:56,751 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 866f4f82d2f771c80496e5b80211b690/B is initiating minor compaction (all files) 2024-11-27T07:26:56,751 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1541): 866f4f82d2f771c80496e5b80211b690/A is initiating minor compaction (all files) 2024-11-27T07:26:56,751 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 866f4f82d2f771c80496e5b80211b690/B in TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:26:56,751 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 866f4f82d2f771c80496e5b80211b690/A in TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:26:56,751 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/9477368a26614e42bf26c794bf89a48e, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/8e76bc29d89d4a09acbc4f2b271d3699, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/de09385b04c94ac89d8307ed809d5079] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp, totalSize=107.5 K 2024-11-27T07:26:56,751 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/47c724e907e14a61865d440ef649fdd1, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/62b5a08980c544bfa2b98228395dbaaa, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/6dc3b53b69aa45c6bc6f5099b3cfd915] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp, totalSize=35.2 K 2024-11-27T07:26:56,751 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:26:56,751 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. files: [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/9477368a26614e42bf26c794bf89a48e, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/8e76bc29d89d4a09acbc4f2b271d3699, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/de09385b04c94ac89d8307ed809d5079] 2024-11-27T07:26:56,751 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 47c724e907e14a61865d440ef649fdd1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732692412250 2024-11-27T07:26:56,751 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9477368a26614e42bf26c794bf89a48e, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732692412247 2024-11-27T07:26:56,752 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 62b5a08980c544bfa2b98228395dbaaa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732692412272 2024-11-27T07:26:56,752 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e76bc29d89d4a09acbc4f2b271d3699, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732692412272 2024-11-27T07:26:56,752 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 6dc3b53b69aa45c6bc6f5099b3cfd915, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732692414436 2024-11-27T07:26:56,752 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting de09385b04c94ac89d8307ed809d5079, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732692414436 2024-11-27T07:26:56,771 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=866f4f82d2f771c80496e5b80211b690] 2024-11-27T07:26:56,772 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 866f4f82d2f771c80496e5b80211b690#B#compaction#450 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:26:56,772 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/B/cafd58d3855f4cafbdc1aedff133a749 is 50, key is test_row_0/B:col10/1732692415071/Put/seqid=0 2024-11-27T07:26:56,785 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112794e4868fd20943939bb0893bb9ad89bb_866f4f82d2f771c80496e5b80211b690 store=[table=TestAcidGuarantees family=A region=866f4f82d2f771c80496e5b80211b690] 2024-11-27T07:26:56,787 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112794e4868fd20943939bb0893bb9ad89bb_866f4f82d2f771c80496e5b80211b690, store=[table=TestAcidGuarantees family=A region=866f4f82d2f771c80496e5b80211b690] 2024-11-27T07:26:56,787 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112794e4868fd20943939bb0893bb9ad89bb_866f4f82d2f771c80496e5b80211b690 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=866f4f82d2f771c80496e5b80211b690] 2024-11-27T07:26:56,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742358_1534 (size=12104) 2024-11-27T07:26:56,793 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/B/cafd58d3855f4cafbdc1aedff133a749 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/cafd58d3855f4cafbdc1aedff133a749 2024-11-27T07:26:56,797 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 866f4f82d2f771c80496e5b80211b690/B of 866f4f82d2f771c80496e5b80211b690 into cafd58d3855f4cafbdc1aedff133a749(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:26:56,797 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 866f4f82d2f771c80496e5b80211b690: 2024-11-27T07:26:56,797 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., storeName=866f4f82d2f771c80496e5b80211b690/B, priority=13, startTime=1732692416750; duration=0sec 2024-11-27T07:26:56,797 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:26:56,797 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 866f4f82d2f771c80496e5b80211b690:B 2024-11-27T07:26:56,797 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:26:56,799 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:26:56,799 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 866f4f82d2f771c80496e5b80211b690/C is initiating minor compaction (all files) 2024-11-27T07:26:56,799 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 866f4f82d2f771c80496e5b80211b690/C in TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:26:56,799 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/90063e32a3444686b89317add3fabd8e, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/399218b9e6e34ceca97f948d02a64294, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/89d1b340afdf4b5fa26093f52d2f109b] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp, totalSize=35.2 K 2024-11-27T07:26:56,799 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 90063e32a3444686b89317add3fabd8e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732692412250 2024-11-27T07:26:56,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742359_1535 (size=4469) 2024-11-27T07:26:56,800 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 399218b9e6e34ceca97f948d02a64294, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732692412272 2024-11-27T07:26:56,800 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 89d1b340afdf4b5fa26093f52d2f109b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732692414436 2024-11-27T07:26:56,806 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 866f4f82d2f771c80496e5b80211b690#C#compaction#452 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:26:56,806 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/C/282ae5b7e80b4f3dbb982d842789eb9b is 50, key is test_row_0/C:col10/1732692415071/Put/seqid=0 2024-11-27T07:26:56,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742360_1536 (size=12104) 2024-11-27T07:26:56,830 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/C/282ae5b7e80b4f3dbb982d842789eb9b as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/282ae5b7e80b4f3dbb982d842789eb9b 2024-11-27T07:26:56,835 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 866f4f82d2f771c80496e5b80211b690/C of 866f4f82d2f771c80496e5b80211b690 into 282ae5b7e80b4f3dbb982d842789eb9b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:26:56,835 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 866f4f82d2f771c80496e5b80211b690: 2024-11-27T07:26:56,835 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., storeName=866f4f82d2f771c80496e5b80211b690/C, priority=13, startTime=1732692416750; duration=0sec 2024-11-27T07:26:56,835 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:26:56,835 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 866f4f82d2f771c80496e5b80211b690:C 2024-11-27T07:26:56,869 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-27T07:26:56,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:26:56,869 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 866f4f82d2f771c80496e5b80211b690 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-27T07:26:56,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112753edc7f41e86414c882c0a0d4f2b4cc6_866f4f82d2f771c80496e5b80211b690 is 50, key is test_row_0/A:col10/1732692415100/Put/seqid=0 2024-11-27T07:26:56,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742361_1537 (size=12154) 2024-11-27T07:26:57,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-27T07:26:57,200 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 866f4f82d2f771c80496e5b80211b690#A#compaction#451 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:26:57,200 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/A/915da50deffb479aaff10b836ed2b8c5 is 175, key is test_row_0/A:col10/1732692415071/Put/seqid=0 2024-11-27T07:26:57,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742362_1538 (size=31058) 2024-11-27T07:26:57,207 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/A/915da50deffb479aaff10b836ed2b8c5 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/915da50deffb479aaff10b836ed2b8c5 2024-11-27T07:26:57,211 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 866f4f82d2f771c80496e5b80211b690/A of 866f4f82d2f771c80496e5b80211b690 into 915da50deffb479aaff10b836ed2b8c5(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:26:57,211 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 866f4f82d2f771c80496e5b80211b690: 2024-11-27T07:26:57,211 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., storeName=866f4f82d2f771c80496e5b80211b690/A, priority=13, startTime=1732692416750; duration=0sec 2024-11-27T07:26:57,211 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:26:57,211 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 866f4f82d2f771c80496e5b80211b690:A 2024-11-27T07:26:57,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 866f4f82d2f771c80496e5b80211b690 2024-11-27T07:26:57,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. as already flushing 2024-11-27T07:26:57,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:57,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59888 deadline: 1732692477265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:57,267 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:57,267 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:57,267 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:57,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:57,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1732692477267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:57,270 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:57,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:57,270 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:57,270 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:57,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59810 deadline: 1732692477267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:57,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:57,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:57,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:57,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:26:57,283 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112753edc7f41e86414c882c0a0d4f2b4cc6_866f4f82d2f771c80496e5b80211b690 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112753edc7f41e86414c882c0a0d4f2b4cc6_866f4f82d2f771c80496e5b80211b690 2024-11-27T07:26:57,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/A/511ad2efdd144292acda8ca66f11fa52, store: [table=TestAcidGuarantees family=A region=866f4f82d2f771c80496e5b80211b690] 2024-11-27T07:26:57,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/A/511ad2efdd144292acda8ca66f11fa52 is 175, key is test_row_0/A:col10/1732692415100/Put/seqid=0 2024-11-27T07:26:57,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742363_1539 (size=30955) 2024-11-27T07:26:57,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:57,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59888 deadline: 1732692477369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:57,370 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:57,370 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:57,370 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:57,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:57,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1732692477379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:57,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:57,380 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:57,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59810 deadline: 1732692477379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:57,380 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:57,380 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:57,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:57,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:57,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:57,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-27T07:26:57,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:57,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59888 deadline: 1732692477579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:57,580 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:57,580 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:57,580 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:57,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:57,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1732692477589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:57,589 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:57,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:57,589 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:57,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:57,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59810 deadline: 1732692477589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:57,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:57,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:57,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:57,687 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=78, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/A/511ad2efdd144292acda8ca66f11fa52 2024-11-27T07:26:57,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/B/e135b9e32b9e4d9da3217b5eff77a71b is 50, key is test_row_0/B:col10/1732692415100/Put/seqid=0 2024-11-27T07:26:57,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742364_1540 (size=12001) 2024-11-27T07:26:57,700 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/B/e135b9e32b9e4d9da3217b5eff77a71b 2024-11-27T07:26:57,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/C/5574eb832c3e4184bcb55c991d6d34da is 50, key is test_row_0/C:col10/1732692415100/Put/seqid=0 2024-11-27T07:26:57,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742365_1541 (size=12001) 2024-11-27T07:26:57,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:57,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59888 deadline: 1732692477889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:57,892 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:57,892 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:57,892 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:57,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:57,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1732692477899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:57,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:57,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:57,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:57,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:57,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59810 deadline: 1732692477899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:57,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:57,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:57,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:58,122 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/C/5574eb832c3e4184bcb55c991d6d34da 2024-11-27T07:26:58,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/A/511ad2efdd144292acda8ca66f11fa52 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/511ad2efdd144292acda8ca66f11fa52 2024-11-27T07:26:58,128 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/511ad2efdd144292acda8ca66f11fa52, entries=150, sequenceid=78, filesize=30.2 K 2024-11-27T07:26:58,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/B/e135b9e32b9e4d9da3217b5eff77a71b as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/e135b9e32b9e4d9da3217b5eff77a71b 2024-11-27T07:26:58,131 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/e135b9e32b9e4d9da3217b5eff77a71b, entries=150, sequenceid=78, filesize=11.7 K 2024-11-27T07:26:58,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/C/5574eb832c3e4184bcb55c991d6d34da as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/5574eb832c3e4184bcb55c991d6d34da 2024-11-27T07:26:58,134 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/5574eb832c3e4184bcb55c991d6d34da, entries=150, sequenceid=78, filesize=11.7 K 2024-11-27T07:26:58,135 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 866f4f82d2f771c80496e5b80211b690 in 1266ms, sequenceid=78, compaction requested=false 2024-11-27T07:26:58,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 866f4f82d2f771c80496e5b80211b690: 2024-11-27T07:26:58,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:26:58,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-27T07:26:58,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-27T07:26:58,138 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-27T07:26:58,138 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7240 sec 2024-11-27T07:26:58,139 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 1.7280 sec 2024-11-27T07:26:58,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 866f4f82d2f771c80496e5b80211b690 2024-11-27T07:26:58,402 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T07:26:58,402 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 866f4f82d2f771c80496e5b80211b690 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-27T07:26:58,407 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411272992cfca95424557bcc1041bb6b64b99_866f4f82d2f771c80496e5b80211b690 is 50, key is test_row_0/A:col10/1732692417266/Put/seqid=0 2024-11-27T07:26:58,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742366_1542 (size=14594) 2024-11-27T07:26:58,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:58,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59888 deadline: 1732692478428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:58,433 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:58,433 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:58,433 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:58,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:58,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1732692478432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:58,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:58,436 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:58,436 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:58,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59810 deadline: 1732692478433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:58,436 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:58,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:58,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:58,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:58,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:58,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59888 deadline: 1732692478539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:58,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:58,541 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:58,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1732692478539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:58,541 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:58,541 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:58,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:58,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:58,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:58,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:58,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59810 deadline: 1732692478539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:58,542 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:58,542 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:58,542 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:58,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-27T07:26:58,569 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T07:26:58,569 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T07:26:58,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-27T07:26:58,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-27T07:26:58,571 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T07:26:58,571 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T07:26:58,571 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T07:26:58,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-27T07:26:58,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-27T07:26:58,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:26:58,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. as already flushing 2024-11-27T07:26:58,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:26:58,723 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:58,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:58,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:58,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:58,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59888 deadline: 1732692478749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:58,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:58,752 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:58,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1732692478749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:58,752 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:58,752 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:58,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:58,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:58,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:58,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:58,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59810 deadline: 1732692478749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:58,752 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:58,752 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:58,752 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:58,815 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:26:58,818 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411272992cfca95424557bcc1041bb6b64b99_866f4f82d2f771c80496e5b80211b690 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411272992cfca95424557bcc1041bb6b64b99_866f4f82d2f771c80496e5b80211b690 2024-11-27T07:26:58,819 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/A/2cb3afa99a6b431d8075d24acff213d0, store: [table=TestAcidGuarantees family=A region=866f4f82d2f771c80496e5b80211b690] 2024-11-27T07:26:58,820 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/A/2cb3afa99a6b431d8075d24acff213d0 is 175, key is test_row_0/A:col10/1732692417266/Put/seqid=0 2024-11-27T07:26:58,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742367_1543 (size=39549) 2024-11-27T07:26:58,823 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/A/2cb3afa99a6b431d8075d24acff213d0 2024-11-27T07:26:58,828 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/B/09e806b8cbb049ddb1a0599a8cb3e161 is 50, key is test_row_0/B:col10/1732692417266/Put/seqid=0 2024-11-27T07:26:58,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742368_1544 (size=12001) 2024-11-27T07:26:58,876 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-27T07:26:58,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:26:58,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. as already flushing 2024-11-27T07:26:58,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:26:58,876 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:58,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:58,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:58,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-27T07:26:59,028 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-27T07:26:59,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:26:59,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. as already flushing 2024-11-27T07:26:59,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:26:59,029 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:59,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:59,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:59,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:59,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59888 deadline: 1732692479059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:59,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:59,061 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:59,061 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:59,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:59,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1732692479059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:59,061 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:59,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59810 deadline: 1732692479059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:59,062 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:59,062 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:59,062 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:59,062 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:59,062 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:59,062 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:59,181 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-27T07:26:59,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:26:59,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. as already flushing 2024-11-27T07:26:59,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:26:59,181 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:59,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:59,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:59,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-27T07:26:59,231 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/B/09e806b8cbb049ddb1a0599a8cb3e161 2024-11-27T07:26:59,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/C/03cb44524dfb4f448891625b5112b4b1 is 50, key is test_row_0/C:col10/1732692417266/Put/seqid=0 2024-11-27T07:26:59,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742369_1545 (size=12001) 2024-11-27T07:26:59,270 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-27T07:26:59,270 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-27T07:26:59,334 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-27T07:26:59,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:26:59,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. as already flushing 2024-11-27T07:26:59,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:26:59,334 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:59,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:59,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:59,486 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-27T07:26:59,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:26:59,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. as already flushing 2024-11-27T07:26:59,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:26:59,487 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:59,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:59,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:59,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:59,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59888 deadline: 1732692479569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:59,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:59,572 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:59,572 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:59,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1732692479569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:59,572 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:59,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:59,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:26:59,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:59,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:59,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59810 deadline: 1732692479569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:26:59,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:59,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:26:59,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:26:59,639 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-27T07:26:59,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:26:59,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. as already flushing 2024-11-27T07:26:59,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:26:59,639 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:59,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:59,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:26:59,640 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/C/03cb44524dfb4f448891625b5112b4b1 2024-11-27T07:26:59,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/A/2cb3afa99a6b431d8075d24acff213d0 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/2cb3afa99a6b431d8075d24acff213d0 2024-11-27T07:26:59,645 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/2cb3afa99a6b431d8075d24acff213d0, entries=200, sequenceid=95, filesize=38.6 K 2024-11-27T07:26:59,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/B/09e806b8cbb049ddb1a0599a8cb3e161 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/09e806b8cbb049ddb1a0599a8cb3e161 2024-11-27T07:26:59,649 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/09e806b8cbb049ddb1a0599a8cb3e161, entries=150, sequenceid=95, filesize=11.7 K 2024-11-27T07:26:59,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/C/03cb44524dfb4f448891625b5112b4b1 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/03cb44524dfb4f448891625b5112b4b1 2024-11-27T07:26:59,652 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/03cb44524dfb4f448891625b5112b4b1, entries=150, sequenceid=95, filesize=11.7 K 2024-11-27T07:26:59,652 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 866f4f82d2f771c80496e5b80211b690 in 1250ms, sequenceid=95, compaction requested=true 2024-11-27T07:26:59,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 866f4f82d2f771c80496e5b80211b690: 2024-11-27T07:26:59,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 866f4f82d2f771c80496e5b80211b690:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T07:26:59,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:26:59,653 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:26:59,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 866f4f82d2f771c80496e5b80211b690:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T07:26:59,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:26:59,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 866f4f82d2f771c80496e5b80211b690:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T07:26:59,653 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:26:59,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:26:59,653 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:26:59,653 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:26:59,653 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 866f4f82d2f771c80496e5b80211b690/A is initiating minor compaction (all files) 2024-11-27T07:26:59,653 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1541): 866f4f82d2f771c80496e5b80211b690/B is initiating minor compaction (all files) 2024-11-27T07:26:59,654 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 866f4f82d2f771c80496e5b80211b690/A in TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:26:59,654 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 866f4f82d2f771c80496e5b80211b690/B in TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:26:59,654 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/915da50deffb479aaff10b836ed2b8c5, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/511ad2efdd144292acda8ca66f11fa52, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/2cb3afa99a6b431d8075d24acff213d0] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp, totalSize=99.2 K 2024-11-27T07:26:59,654 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/cafd58d3855f4cafbdc1aedff133a749, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/e135b9e32b9e4d9da3217b5eff77a71b, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/09e806b8cbb049ddb1a0599a8cb3e161] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp, totalSize=35.3 K 2024-11-27T07:26:59,654 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:26:59,654 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. files: [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/915da50deffb479aaff10b836ed2b8c5, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/511ad2efdd144292acda8ca66f11fa52, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/2cb3afa99a6b431d8075d24acff213d0] 2024-11-27T07:26:59,654 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting cafd58d3855f4cafbdc1aedff133a749, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732692414436 2024-11-27T07:26:59,654 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 915da50deffb479aaff10b836ed2b8c5, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732692414436 2024-11-27T07:26:59,654 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting e135b9e32b9e4d9da3217b5eff77a71b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732692415097 2024-11-27T07:26:59,654 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 511ad2efdd144292acda8ca66f11fa52, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732692415097 2024-11-27T07:26:59,654 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09e806b8cbb049ddb1a0599a8cb3e161, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732692417265 2024-11-27T07:26:59,654 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 2cb3afa99a6b431d8075d24acff213d0, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732692417262 2024-11-27T07:26:59,658 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=866f4f82d2f771c80496e5b80211b690] 2024-11-27T07:26:59,659 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 866f4f82d2f771c80496e5b80211b690#B#compaction#459 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:26:59,660 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/B/cbaab1e139f947d4a24e68e1e3d387a3 is 50, key is test_row_0/B:col10/1732692417266/Put/seqid=0 2024-11-27T07:26:59,662 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112767fcec7882d3492ea7af1f3ffb1ddbd4_866f4f82d2f771c80496e5b80211b690 store=[table=TestAcidGuarantees family=A region=866f4f82d2f771c80496e5b80211b690] 2024-11-27T07:26:59,664 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112767fcec7882d3492ea7af1f3ffb1ddbd4_866f4f82d2f771c80496e5b80211b690, store=[table=TestAcidGuarantees family=A region=866f4f82d2f771c80496e5b80211b690] 2024-11-27T07:26:59,664 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112767fcec7882d3492ea7af1f3ffb1ddbd4_866f4f82d2f771c80496e5b80211b690 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=866f4f82d2f771c80496e5b80211b690] 2024-11-27T07:26:59,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742370_1546 (size=12207) 2024-11-27T07:26:59,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742371_1547 (size=4469) 2024-11-27T07:26:59,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-27T07:26:59,791 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-27T07:26:59,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:26:59,791 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 866f4f82d2f771c80496e5b80211b690 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-27T07:26:59,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411278a6c0f83f88b4ccb9a436646b0ba667b_866f4f82d2f771c80496e5b80211b690 is 50, key is test_row_0/A:col10/1732692418432/Put/seqid=0 2024-11-27T07:26:59,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742372_1548 (size=12154) 2024-11-27T07:26:59,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:26:59,805 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411278a6c0f83f88b4ccb9a436646b0ba667b_866f4f82d2f771c80496e5b80211b690 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411278a6c0f83f88b4ccb9a436646b0ba667b_866f4f82d2f771c80496e5b80211b690 2024-11-27T07:26:59,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/A/e06dd5d0cc544e699e9eb27948f1e155, store: [table=TestAcidGuarantees family=A region=866f4f82d2f771c80496e5b80211b690] 2024-11-27T07:26:59,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/A/e06dd5d0cc544e699e9eb27948f1e155 is 175, key is test_row_0/A:col10/1732692418432/Put/seqid=0 2024-11-27T07:26:59,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742373_1549 (size=30955) 2024-11-27T07:27:00,068 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 866f4f82d2f771c80496e5b80211b690#A#compaction#460 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:27:00,068 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/A/eaa36318d26742adbeb0ffc775c6c069 is 175, key is test_row_0/A:col10/1732692417266/Put/seqid=0 2024-11-27T07:27:00,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742374_1550 (size=31161) 2024-11-27T07:27:00,074 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/B/cbaab1e139f947d4a24e68e1e3d387a3 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/cbaab1e139f947d4a24e68e1e3d387a3 2024-11-27T07:27:00,076 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/A/eaa36318d26742adbeb0ffc775c6c069 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/eaa36318d26742adbeb0ffc775c6c069 2024-11-27T07:27:00,078 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 866f4f82d2f771c80496e5b80211b690/B of 866f4f82d2f771c80496e5b80211b690 into cbaab1e139f947d4a24e68e1e3d387a3(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:27:00,078 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 866f4f82d2f771c80496e5b80211b690: 2024-11-27T07:27:00,078 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., storeName=866f4f82d2f771c80496e5b80211b690/B, priority=13, startTime=1732692419653; duration=0sec 2024-11-27T07:27:00,078 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:27:00,079 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 866f4f82d2f771c80496e5b80211b690:B 2024-11-27T07:27:00,079 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:27:00,080 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 866f4f82d2f771c80496e5b80211b690/A of 866f4f82d2f771c80496e5b80211b690 into eaa36318d26742adbeb0ffc775c6c069(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:27:00,080 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 866f4f82d2f771c80496e5b80211b690: 2024-11-27T07:27:00,080 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:27:00,080 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., storeName=866f4f82d2f771c80496e5b80211b690/A, priority=13, startTime=1732692419653; duration=0sec 2024-11-27T07:27:00,080 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1541): 866f4f82d2f771c80496e5b80211b690/C is initiating minor compaction (all files) 2024-11-27T07:27:00,080 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 866f4f82d2f771c80496e5b80211b690/C in TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:00,080 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:27:00,080 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 866f4f82d2f771c80496e5b80211b690:A 2024-11-27T07:27:00,080 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/282ae5b7e80b4f3dbb982d842789eb9b, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/5574eb832c3e4184bcb55c991d6d34da, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/03cb44524dfb4f448891625b5112b4b1] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp, totalSize=35.3 K 2024-11-27T07:27:00,081 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 282ae5b7e80b4f3dbb982d842789eb9b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732692414436 2024-11-27T07:27:00,081 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5574eb832c3e4184bcb55c991d6d34da, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732692415097 2024-11-27T07:27:00,081 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03cb44524dfb4f448891625b5112b4b1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732692417265 2024-11-27T07:27:00,086 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 866f4f82d2f771c80496e5b80211b690#C#compaction#462 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:27:00,086 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/C/61bddf22f49844bb81de3b8c71f890d9 is 50, key is test_row_0/C:col10/1732692417266/Put/seqid=0 2024-11-27T07:27:00,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742375_1551 (size=12207) 2024-11-27T07:27:00,093 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/C/61bddf22f49844bb81de3b8c71f890d9 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/61bddf22f49844bb81de3b8c71f890d9 2024-11-27T07:27:00,097 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 866f4f82d2f771c80496e5b80211b690/C of 866f4f82d2f771c80496e5b80211b690 into 61bddf22f49844bb81de3b8c71f890d9(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:27:00,097 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 866f4f82d2f771c80496e5b80211b690: 2024-11-27T07:27:00,097 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., storeName=866f4f82d2f771c80496e5b80211b690/C, priority=13, startTime=1732692419653; duration=0sec 2024-11-27T07:27:00,097 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:27:00,097 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 866f4f82d2f771c80496e5b80211b690:C 2024-11-27T07:27:00,209 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=115, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/A/e06dd5d0cc544e699e9eb27948f1e155 2024-11-27T07:27:00,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/B/757e19a53aaa4b5cb1e00a06f6d83dfd is 50, key is test_row_0/B:col10/1732692418432/Put/seqid=0 2024-11-27T07:27:00,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742376_1552 (size=12001) 2024-11-27T07:27:00,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 866f4f82d2f771c80496e5b80211b690 2024-11-27T07:27:00,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. as already flushing 2024-11-27T07:27:00,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:00,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1732692480509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:00,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:00,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:00,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:00,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:00,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59860 deadline: 1732692480509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:00,512 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 289954c9202d,42117,1732692259671 for 'test_row_0' in 866f4f82d2f771c80496e5b80211b690 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8239 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T07:27:00,513 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:00,513 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:00,513 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:00,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:00,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59888 deadline: 1732692480579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:00,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:00,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1732692480579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:00,581 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 289954c9202d,42117,1732692259671 for 'test_row_2' in 866f4f82d2f771c80496e5b80211b690 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T07:27:00,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:00,581 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:00,581 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:00,581 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:00,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59810 deadline: 1732692480579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:00,581 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 289954c9202d,42117,1732692259671 for 'test_row_2' in 866f4f82d2f771c80496e5b80211b690 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2149 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T07:27:00,582 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:00,582 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:00,582 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:00,582 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 289954c9202d,42117,1732692259671 for 'test_row_1' in 866f4f82d2f771c80496e5b80211b690 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2149 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T07:27:00,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:00,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:00,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:00,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:00,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1732692480618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:00,621 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:00,621 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:00,621 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:00,628 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/B/757e19a53aaa4b5cb1e00a06f6d83dfd 2024-11-27T07:27:00,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/C/74d6142d0abe4be8854b9790405fad78 is 50, key is test_row_0/C:col10/1732692418432/Put/seqid=0 2024-11-27T07:27:00,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742377_1553 (size=12001) 2024-11-27T07:27:00,638 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/C/74d6142d0abe4be8854b9790405fad78 2024-11-27T07:27:00,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/A/e06dd5d0cc544e699e9eb27948f1e155 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/e06dd5d0cc544e699e9eb27948f1e155 2024-11-27T07:27:00,645 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/e06dd5d0cc544e699e9eb27948f1e155, entries=150, sequenceid=115, filesize=30.2 K 2024-11-27T07:27:00,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/B/757e19a53aaa4b5cb1e00a06f6d83dfd as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/757e19a53aaa4b5cb1e00a06f6d83dfd 2024-11-27T07:27:00,649 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/757e19a53aaa4b5cb1e00a06f6d83dfd, entries=150, sequenceid=115, filesize=11.7 K 2024-11-27T07:27:00,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/C/74d6142d0abe4be8854b9790405fad78 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/74d6142d0abe4be8854b9790405fad78 2024-11-27T07:27:00,653 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/74d6142d0abe4be8854b9790405fad78, entries=150, sequenceid=115, filesize=11.7 K 2024-11-27T07:27:00,654 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 866f4f82d2f771c80496e5b80211b690 in 863ms, sequenceid=115, compaction requested=false 2024-11-27T07:27:00,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 866f4f82d2f771c80496e5b80211b690: 2024-11-27T07:27:00,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:00,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-27T07:27:00,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-27T07:27:00,657 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-27T07:27:00,657 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0840 sec 2024-11-27T07:27:00,659 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 2.0890 sec 2024-11-27T07:27:00,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-27T07:27:00,729 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T07:27:00,730 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T07:27:00,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-27T07:27:00,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-27T07:27:00,731 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T07:27:00,732 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T07:27:00,732 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T07:27:00,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 866f4f82d2f771c80496e5b80211b690 2024-11-27T07:27:00,830 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T07:27:00,831 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 866f4f82d2f771c80496e5b80211b690 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-27T07:27:00,836 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411278e0b6996bf26410991d786d74ebc408d_866f4f82d2f771c80496e5b80211b690 is 50, key is test_row_0/A:col10/1732692420491/Put/seqid=0 2024-11-27T07:27:00,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-27T07:27:00,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742378_1554 (size=14744) 2024-11-27T07:27:00,883 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-27T07:27:00,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:00,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. as already flushing 2024-11-27T07:27:00,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:00,884 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:27:00,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:27:00,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:27:00,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:00,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1732692480886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:00,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:00,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:00,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:01,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:01,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1732692480999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:01,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:01,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:01,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:01,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-27T07:27:01,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:01,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. as already flushing 2024-11-27T07:27:01,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:01,036 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:27:01,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:27:01,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:27:01,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-27T07:27:01,188 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-27T07:27:01,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:01,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. as already flushing 2024-11-27T07:27:01,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:01,189 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:27:01,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:27:01,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:27:01,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:01,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1732692481209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:01,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:01,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:01,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:01,259 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:27:01,261 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411278e0b6996bf26410991d786d74ebc408d_866f4f82d2f771c80496e5b80211b690 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411278e0b6996bf26410991d786d74ebc408d_866f4f82d2f771c80496e5b80211b690 2024-11-27T07:27:01,262 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/A/64e3b5df06f14459b1777e302fa41f7f, store: [table=TestAcidGuarantees family=A region=866f4f82d2f771c80496e5b80211b690] 2024-11-27T07:27:01,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/A/64e3b5df06f14459b1777e302fa41f7f is 175, key is test_row_0/A:col10/1732692420491/Put/seqid=0 2024-11-27T07:27:01,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742379_1555 (size=39699) 2024-11-27T07:27:01,341 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-27T07:27:01,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:01,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. as already flushing 2024-11-27T07:27:01,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:01,341 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:27:01,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:27:01,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:27:01,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-27T07:27:01,494 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-27T07:27:01,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:01,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. as already flushing 2024-11-27T07:27:01,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:01,494 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:27:01,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:27:01,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:27:01,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:01,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1732692481519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:01,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:01,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:01,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:01,646 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-27T07:27:01,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:01,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. as already flushing 2024-11-27T07:27:01,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:01,647 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:27:01,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:27:01,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:27:01,703 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=135, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/A/64e3b5df06f14459b1777e302fa41f7f 2024-11-27T07:27:01,708 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/B/e547e6b683204b51b63f48bd5b9d767c is 50, key is test_row_0/B:col10/1732692420491/Put/seqid=0 2024-11-27T07:27:01,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742380_1556 (size=12101) 2024-11-27T07:27:01,798 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-27T07:27:01,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:01,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. as already flushing 2024-11-27T07:27:01,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:01,799 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:27:01,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:27:01,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:27:01,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-27T07:27:01,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-27T07:27:01,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:01,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. as already flushing 2024-11-27T07:27:01,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:01,951 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:27:01,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:27:01,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:27:02,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:02,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1732692482029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:02,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:02,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:02,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:02,103 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-27T07:27:02,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:02,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. as already flushing 2024-11-27T07:27:02,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:02,104 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:27:02,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:27:02,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:27:02,110 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/B/e547e6b683204b51b63f48bd5b9d767c 2024-11-27T07:27:02,117 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/C/c969d12d2b1a4d1b95d3c2111ab4f92b is 50, key is test_row_0/C:col10/1732692420491/Put/seqid=0 2024-11-27T07:27:02,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742381_1557 (size=12101) 2024-11-27T07:27:02,256 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-27T07:27:02,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:02,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. as already flushing 2024-11-27T07:27:02,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:02,256 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:27:02,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:27:02,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:27:02,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-27T07:27:02,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:02,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. as already flushing 2024-11-27T07:27:02,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:02,409 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:27:02,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:27:02,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:27:02,520 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/C/c969d12d2b1a4d1b95d3c2111ab4f92b 2024-11-27T07:27:02,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/A/64e3b5df06f14459b1777e302fa41f7f as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/64e3b5df06f14459b1777e302fa41f7f 2024-11-27T07:27:02,527 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/64e3b5df06f14459b1777e302fa41f7f, entries=200, sequenceid=135, filesize=38.8 K 2024-11-27T07:27:02,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/B/e547e6b683204b51b63f48bd5b9d767c as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/e547e6b683204b51b63f48bd5b9d767c 2024-11-27T07:27:02,536 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/e547e6b683204b51b63f48bd5b9d767c, entries=150, sequenceid=135, filesize=11.8 K 2024-11-27T07:27:02,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/C/c969d12d2b1a4d1b95d3c2111ab4f92b as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/c969d12d2b1a4d1b95d3c2111ab4f92b 2024-11-27T07:27:02,541 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/c969d12d2b1a4d1b95d3c2111ab4f92b, entries=150, sequenceid=135, filesize=11.8 K 2024-11-27T07:27:02,541 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 866f4f82d2f771c80496e5b80211b690 in 1711ms, sequenceid=135, compaction requested=true 2024-11-27T07:27:02,541 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 866f4f82d2f771c80496e5b80211b690: 2024-11-27T07:27:02,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 866f4f82d2f771c80496e5b80211b690:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T07:27:02,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:27:02,542 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:27:02,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 866f4f82d2f771c80496e5b80211b690:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T07:27:02,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:27:02,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 866f4f82d2f771c80496e5b80211b690:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T07:27:02,542 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:27:02,542 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:27:02,542 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:27:02,542 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101815 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:27:02,542 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 866f4f82d2f771c80496e5b80211b690/B is initiating minor compaction (all files) 2024-11-27T07:27:02,542 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1541): 866f4f82d2f771c80496e5b80211b690/A is initiating minor compaction (all files) 2024-11-27T07:27:02,543 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 866f4f82d2f771c80496e5b80211b690/B in TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:02,543 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 866f4f82d2f771c80496e5b80211b690/A in TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:02,543 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/eaa36318d26742adbeb0ffc775c6c069, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/e06dd5d0cc544e699e9eb27948f1e155, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/64e3b5df06f14459b1777e302fa41f7f] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp, totalSize=99.4 K 2024-11-27T07:27:02,543 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/cbaab1e139f947d4a24e68e1e3d387a3, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/757e19a53aaa4b5cb1e00a06f6d83dfd, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/e547e6b683204b51b63f48bd5b9d767c] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp, totalSize=35.5 K 2024-11-27T07:27:02,543 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:02,543 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. files: [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/eaa36318d26742adbeb0ffc775c6c069, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/e06dd5d0cc544e699e9eb27948f1e155, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/64e3b5df06f14459b1777e302fa41f7f] 2024-11-27T07:27:02,543 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting cbaab1e139f947d4a24e68e1e3d387a3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732692417265 2024-11-27T07:27:02,543 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting eaa36318d26742adbeb0ffc775c6c069, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732692417265 2024-11-27T07:27:02,543 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 757e19a53aaa4b5cb1e00a06f6d83dfd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732692418427 2024-11-27T07:27:02,543 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting e06dd5d0cc544e699e9eb27948f1e155, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732692418427 2024-11-27T07:27:02,544 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64e3b5df06f14459b1777e302fa41f7f, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732692420491 2024-11-27T07:27:02,544 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting e547e6b683204b51b63f48bd5b9d767c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732692420491 2024-11-27T07:27:02,548 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=866f4f82d2f771c80496e5b80211b690] 2024-11-27T07:27:02,554 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411274113a55864fa44deb9e527cb47b29e84_866f4f82d2f771c80496e5b80211b690 store=[table=TestAcidGuarantees family=A region=866f4f82d2f771c80496e5b80211b690] 2024-11-27T07:27:02,554 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 866f4f82d2f771c80496e5b80211b690#B#compaction#469 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:27:02,555 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/B/e11af2ddf64e420692cca2bfa9b88a47 is 50, key is test_row_0/B:col10/1732692420491/Put/seqid=0 2024-11-27T07:27:02,556 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411274113a55864fa44deb9e527cb47b29e84_866f4f82d2f771c80496e5b80211b690, store=[table=TestAcidGuarantees family=A region=866f4f82d2f771c80496e5b80211b690] 2024-11-27T07:27:02,556 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411274113a55864fa44deb9e527cb47b29e84_866f4f82d2f771c80496e5b80211b690 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=866f4f82d2f771c80496e5b80211b690] 2024-11-27T07:27:02,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742382_1558 (size=12409) 2024-11-27T07:27:02,561 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-27T07:27:02,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:02,561 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 866f4f82d2f771c80496e5b80211b690 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-27T07:27:02,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742383_1559 (size=4469) 2024-11-27T07:27:02,563 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 866f4f82d2f771c80496e5b80211b690#A#compaction#468 average throughput is 1.63 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:27:02,563 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/A/5029d8a2d3714ade96b87b46210a241f is 175, key is test_row_0/A:col10/1732692420491/Put/seqid=0 2024-11-27T07:27:02,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127fa197227c584409aa1cfb9b14cb69645_866f4f82d2f771c80496e5b80211b690 is 50, key is test_row_0/A:col10/1732692420880/Put/seqid=0 2024-11-27T07:27:02,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742384_1560 (size=31363) 2024-11-27T07:27:02,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742385_1561 (size=12304) 2024-11-27T07:27:02,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 866f4f82d2f771c80496e5b80211b690 2024-11-27T07:27:02,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. as already flushing 2024-11-27T07:27:02,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:02,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1732692482622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:02,626 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:02,626 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:02,626 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:02,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:02,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59888 deadline: 1732692482623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:02,627 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:02,627 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:02,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:02,627 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:02,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59810 deadline: 1732692482624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:02,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:02,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:02,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:02,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:02,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1732692482729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:02,730 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:02,730 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:02,730 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:02,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:02,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59888 deadline: 1732692482739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:02,740 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:02,740 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:02,740 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:02,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:02,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59810 deadline: 1732692482739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:02,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:02,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:02,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:02,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-27T07:27:02,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:02,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1732692482939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:02,942 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:02,942 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:02,943 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:02,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:02,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59888 deadline: 1732692482949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:02,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:02,951 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:02,951 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:02,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59810 deadline: 1732692482949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:02,951 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:02,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:02,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:02,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:02,967 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/B/e11af2ddf64e420692cca2bfa9b88a47 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/e11af2ddf64e420692cca2bfa9b88a47 2024-11-27T07:27:02,971 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 866f4f82d2f771c80496e5b80211b690/B of 866f4f82d2f771c80496e5b80211b690 into e11af2ddf64e420692cca2bfa9b88a47(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:27:02,971 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 866f4f82d2f771c80496e5b80211b690: 2024-11-27T07:27:02,971 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., storeName=866f4f82d2f771c80496e5b80211b690/B, priority=13, startTime=1732692422542; duration=0sec 2024-11-27T07:27:02,971 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:27:02,971 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 866f4f82d2f771c80496e5b80211b690:B 2024-11-27T07:27:02,971 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:27:02,971 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:27:02,971 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 866f4f82d2f771c80496e5b80211b690/C is initiating minor compaction (all files) 2024-11-27T07:27:02,972 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 866f4f82d2f771c80496e5b80211b690/C in TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:02,972 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/61bddf22f49844bb81de3b8c71f890d9, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/74d6142d0abe4be8854b9790405fad78, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/c969d12d2b1a4d1b95d3c2111ab4f92b] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp, totalSize=35.5 K 2024-11-27T07:27:02,972 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 61bddf22f49844bb81de3b8c71f890d9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732692417265 2024-11-27T07:27:02,972 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 74d6142d0abe4be8854b9790405fad78, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732692418427 2024-11-27T07:27:02,972 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting c969d12d2b1a4d1b95d3c2111ab4f92b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732692420491 2024-11-27T07:27:02,977 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 866f4f82d2f771c80496e5b80211b690#C#compaction#471 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:27:02,978 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/C/34057b4b5fa14b1f9047b128bfcbd4ca is 50, key is test_row_0/C:col10/1732692420491/Put/seqid=0 2024-11-27T07:27:02,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742386_1562 (size=12409) 2024-11-27T07:27:02,983 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/A/5029d8a2d3714ade96b87b46210a241f as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/5029d8a2d3714ade96b87b46210a241f 2024-11-27T07:27:02,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:27:02,987 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/C/34057b4b5fa14b1f9047b128bfcbd4ca as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/34057b4b5fa14b1f9047b128bfcbd4ca 2024-11-27T07:27:02,988 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127fa197227c584409aa1cfb9b14cb69645_866f4f82d2f771c80496e5b80211b690 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127fa197227c584409aa1cfb9b14cb69645_866f4f82d2f771c80496e5b80211b690 2024-11-27T07:27:02,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/A/70a9d5eeb86748ea85e80847ade7a471, store: [table=TestAcidGuarantees family=A region=866f4f82d2f771c80496e5b80211b690] 2024-11-27T07:27:02,989 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 866f4f82d2f771c80496e5b80211b690/A of 866f4f82d2f771c80496e5b80211b690 into 5029d8a2d3714ade96b87b46210a241f(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:27:02,989 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 866f4f82d2f771c80496e5b80211b690: 2024-11-27T07:27:02,989 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., storeName=866f4f82d2f771c80496e5b80211b690/A, priority=13, startTime=1732692422541; duration=0sec 2024-11-27T07:27:02,989 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:27:02,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/A/70a9d5eeb86748ea85e80847ade7a471 is 175, key is test_row_0/A:col10/1732692420880/Put/seqid=0 2024-11-27T07:27:02,989 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 866f4f82d2f771c80496e5b80211b690:A 2024-11-27T07:27:02,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742387_1563 (size=31105) 2024-11-27T07:27:02,999 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 866f4f82d2f771c80496e5b80211b690/C of 866f4f82d2f771c80496e5b80211b690 into 34057b4b5fa14b1f9047b128bfcbd4ca(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:27:02,999 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=154, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/A/70a9d5eeb86748ea85e80847ade7a471 2024-11-27T07:27:02,999 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 866f4f82d2f771c80496e5b80211b690: 2024-11-27T07:27:02,999 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., storeName=866f4f82d2f771c80496e5b80211b690/C, priority=13, startTime=1732692422542; duration=0sec 2024-11-27T07:27:02,999 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:27:02,999 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 866f4f82d2f771c80496e5b80211b690:C 2024-11-27T07:27:03,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/B/0c5c8049f7cf4a259df080f194eb4cc0 is 50, key is test_row_0/B:col10/1732692420880/Put/seqid=0 2024-11-27T07:27:03,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742388_1564 (size=12151) 2024-11-27T07:27:03,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:03,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1732692483039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:03,041 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 289954c9202d,42117,1732692259671 for 'test_row_1' in 866f4f82d2f771c80496e5b80211b690 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T07:27:03,042 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:03,042 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:03,042 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:03,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:03,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1732692483249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:03,251 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:03,251 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:03,251 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:03,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:03,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59888 deadline: 1732692483259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:03,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:03,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59810 deadline: 1732692483259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:03,260 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:03,260 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:03,260 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:03,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:03,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:03,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:03,410 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/B/0c5c8049f7cf4a259df080f194eb4cc0 2024-11-27T07:27:03,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/C/1cab719c6f6e49b0ab0ad8f65f2e766d is 50, key is test_row_0/C:col10/1732692420880/Put/seqid=0 2024-11-27T07:27:03,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742389_1565 (size=12151) 2024-11-27T07:27:03,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:03,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1732692483759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:03,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:03,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:03,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:03,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:03,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59888 deadline: 1732692483769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:03,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:03,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59810 deadline: 1732692483769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:03,771 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:03,771 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:03,771 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:03,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:03,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:03,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:03,820 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/C/1cab719c6f6e49b0ab0ad8f65f2e766d 2024-11-27T07:27:03,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/A/70a9d5eeb86748ea85e80847ade7a471 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/70a9d5eeb86748ea85e80847ade7a471 2024-11-27T07:27:03,826 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/70a9d5eeb86748ea85e80847ade7a471, entries=150, sequenceid=154, filesize=30.4 K 2024-11-27T07:27:03,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/B/0c5c8049f7cf4a259df080f194eb4cc0 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/0c5c8049f7cf4a259df080f194eb4cc0 2024-11-27T07:27:03,830 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/0c5c8049f7cf4a259df080f194eb4cc0, entries=150, sequenceid=154, filesize=11.9 K 2024-11-27T07:27:03,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/C/1cab719c6f6e49b0ab0ad8f65f2e766d as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/1cab719c6f6e49b0ab0ad8f65f2e766d 2024-11-27T07:27:03,834 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/1cab719c6f6e49b0ab0ad8f65f2e766d, entries=150, sequenceid=154, filesize=11.9 K 2024-11-27T07:27:03,835 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 866f4f82d2f771c80496e5b80211b690 in 1273ms, sequenceid=154, compaction requested=false 2024-11-27T07:27:03,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 866f4f82d2f771c80496e5b80211b690: 2024-11-27T07:27:03,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:03,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-27T07:27:03,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-27T07:27:03,839 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-27T07:27:03,839 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1050 sec 2024-11-27T07:27:03,840 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 3.1090 sec 2024-11-27T07:27:04,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 866f4f82d2f771c80496e5b80211b690 2024-11-27T07:27:04,771 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T07:27:04,771 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 866f4f82d2f771c80496e5b80211b690 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-27T07:27:04,777 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127382ed256c44f4ab4af59457cc5de1273_866f4f82d2f771c80496e5b80211b690 is 50, key is test_row_0/A:col10/1732692422621/Put/seqid=0 2024-11-27T07:27:04,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742390_1566 (size=12304) 2024-11-27T07:27:04,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:04,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59810 deadline: 1732692484801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:04,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:04,805 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:04,805 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:04,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1732692484802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:04,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:04,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:04,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:04,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:04,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:04,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59888 deadline: 1732692484802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:04,806 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:04,806 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:04,806 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:04,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-27T07:27:04,900 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T07:27:04,900 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T07:27:04,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-27T07:27:04,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-27T07:27:04,901 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T07:27:04,902 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T07:27:04,902 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T07:27:04,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:04,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59810 deadline: 1732692484909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:04,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:04,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:04,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:04,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:04,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59888 deadline: 1732692484909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:04,911 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:04,911 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:04,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:04,911 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:04,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1732692484909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:04,911 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:04,911 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:04,911 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:05,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-27T07:27:05,054 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-27T07:27:05,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:05,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. as already flushing 2024-11-27T07:27:05,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:05,054 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:27:05,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:27:05,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:27:05,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:05,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1732692485059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:05,062 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 289954c9202d,42117,1732692259671 for 'test_row_1' in 866f4f82d2f771c80496e5b80211b690 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T07:27:05,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:05,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:05,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:05,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:05,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59810 deadline: 1732692485119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:05,121 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:05,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:05,121 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:05,121 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:05,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59888 deadline: 1732692485119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:05,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:05,122 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:05,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1732692485119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:05,122 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:05,122 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:05,122 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:05,122 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:05,122 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:05,181 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:27:05,183 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127382ed256c44f4ab4af59457cc5de1273_866f4f82d2f771c80496e5b80211b690 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127382ed256c44f4ab4af59457cc5de1273_866f4f82d2f771c80496e5b80211b690 2024-11-27T07:27:05,184 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/A/c6bb39fe6d88419c82352c33b9f2a8d3, store: [table=TestAcidGuarantees family=A region=866f4f82d2f771c80496e5b80211b690] 2024-11-27T07:27:05,184 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/A/c6bb39fe6d88419c82352c33b9f2a8d3 is 175, key is test_row_0/A:col10/1732692422621/Put/seqid=0 2024-11-27T07:27:05,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742391_1567 (size=31105) 2024-11-27T07:27:05,188 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=175, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/A/c6bb39fe6d88419c82352c33b9f2a8d3 2024-11-27T07:27:05,193 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/B/c2b926b158714d33b221d97f29e4c6f7 is 50, key is test_row_0/B:col10/1732692422621/Put/seqid=0 2024-11-27T07:27:05,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742392_1568 (size=12151) 2024-11-27T07:27:05,206 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-27T07:27:05,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:05,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. as already flushing 2024-11-27T07:27:05,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:05,207 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:27:05,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:27:05,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:27:05,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-27T07:27:05,358 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-27T07:27:05,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:05,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. as already flushing 2024-11-27T07:27:05,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:05,359 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:27:05,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:27:05,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:27:05,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:05,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:05,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59810 deadline: 1732692485429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:05,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59888 deadline: 1732692485429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:05,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:05,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1732692485429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:05,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:05,432 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:05,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:05,432 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:05,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:05,432 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:05,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:05,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:05,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:05,511 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-27T07:27:05,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:05,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. as already flushing 2024-11-27T07:27:05,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:05,511 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:27:05,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:27:05,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:27:05,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-27T07:27:05,596 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/B/c2b926b158714d33b221d97f29e4c6f7 2024-11-27T07:27:05,601 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/C/616de2be0dd74b7086c957ae21596669 is 50, key is test_row_0/C:col10/1732692422621/Put/seqid=0 2024-11-27T07:27:05,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742393_1569 (size=12151) 2024-11-27T07:27:05,664 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-27T07:27:05,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:05,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. as already flushing 2024-11-27T07:27:05,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:05,664 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:27:05,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:27:05,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:27:05,816 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-27T07:27:05,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:05,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. as already flushing 2024-11-27T07:27:05,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:05,817 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:27:05,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:27:05,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:27:05,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:05,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59888 deadline: 1732692485940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:05,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:05,941 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:05,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59810 deadline: 1732692485940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:05,941 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:05,941 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:05,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:05,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1732692485940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:05,942 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:05,942 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:05,942 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:05,942 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:05,942 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:05,942 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:05,971 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-27T07:27:05,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:05,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. as already flushing 2024-11-27T07:27:05,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:05,971 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:27:05,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:27:05,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:27:06,012 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/C/616de2be0dd74b7086c957ae21596669 2024-11-27T07:27:06,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/A/c6bb39fe6d88419c82352c33b9f2a8d3 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/c6bb39fe6d88419c82352c33b9f2a8d3 2024-11-27T07:27:06,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-27T07:27:06,045 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/c6bb39fe6d88419c82352c33b9f2a8d3, entries=150, sequenceid=175, filesize=30.4 K 2024-11-27T07:27:06,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/B/c2b926b158714d33b221d97f29e4c6f7 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/c2b926b158714d33b221d97f29e4c6f7 2024-11-27T07:27:06,082 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/c2b926b158714d33b221d97f29e4c6f7, entries=150, sequenceid=175, filesize=11.9 K 2024-11-27T07:27:06,092 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/C/616de2be0dd74b7086c957ae21596669 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/616de2be0dd74b7086c957ae21596669 2024-11-27T07:27:06,129 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-27T07:27:06,129 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/616de2be0dd74b7086c957ae21596669, entries=150, sequenceid=175, filesize=11.9 K 2024-11-27T07:27:06,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:06,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. as already flushing 2024-11-27T07:27:06,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:06,130 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:27:06,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:27:06,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:27:06,140 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=107.34 KB/109920 for 866f4f82d2f771c80496e5b80211b690 in 1369ms, sequenceid=175, compaction requested=true 2024-11-27T07:27:06,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 866f4f82d2f771c80496e5b80211b690: 2024-11-27T07:27:06,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 866f4f82d2f771c80496e5b80211b690:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T07:27:06,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:27:06,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 866f4f82d2f771c80496e5b80211b690:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T07:27:06,140 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:27:06,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:27:06,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 866f4f82d2f771c80496e5b80211b690:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T07:27:06,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-27T07:27:06,141 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:27:06,153 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:27:06,153 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 866f4f82d2f771c80496e5b80211b690/B is initiating minor compaction (all files) 2024-11-27T07:27:06,153 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 866f4f82d2f771c80496e5b80211b690/B in TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:06,153 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/e11af2ddf64e420692cca2bfa9b88a47, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/0c5c8049f7cf4a259df080f194eb4cc0, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/c2b926b158714d33b221d97f29e4c6f7] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp, totalSize=35.9 K 2024-11-27T07:27:06,153 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93573 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:27:06,153 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1541): 866f4f82d2f771c80496e5b80211b690/A is initiating minor compaction (all files) 2024-11-27T07:27:06,153 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 866f4f82d2f771c80496e5b80211b690/A in TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:06,154 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/5029d8a2d3714ade96b87b46210a241f, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/70a9d5eeb86748ea85e80847ade7a471, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/c6bb39fe6d88419c82352c33b9f2a8d3] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp, totalSize=91.4 K 2024-11-27T07:27:06,154 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:06,154 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. files: [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/5029d8a2d3714ade96b87b46210a241f, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/70a9d5eeb86748ea85e80847ade7a471, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/c6bb39fe6d88419c82352c33b9f2a8d3] 2024-11-27T07:27:06,156 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting e11af2ddf64e420692cca2bfa9b88a47, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732692420491 2024-11-27T07:27:06,157 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5029d8a2d3714ade96b87b46210a241f, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732692420491 2024-11-27T07:27:06,161 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c5c8049f7cf4a259df080f194eb4cc0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732692420865 2024-11-27T07:27:06,161 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70a9d5eeb86748ea85e80847ade7a471, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732692420865 2024-11-27T07:27:06,164 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting c2b926b158714d33b221d97f29e4c6f7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1732692422621 2024-11-27T07:27:06,165 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6bb39fe6d88419c82352c33b9f2a8d3, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1732692422621 2024-11-27T07:27:06,209 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=866f4f82d2f771c80496e5b80211b690] 2024-11-27T07:27:06,211 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 866f4f82d2f771c80496e5b80211b690#B#compaction#478 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:27:06,212 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/B/f546472b7e014255ab9c968d9fa3c509 is 50, key is test_row_0/B:col10/1732692422621/Put/seqid=0 2024-11-27T07:27:06,227 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127bd40f8c31ebd49e6b84fa0d37d90679f_866f4f82d2f771c80496e5b80211b690 store=[table=TestAcidGuarantees family=A region=866f4f82d2f771c80496e5b80211b690] 2024-11-27T07:27:06,233 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127bd40f8c31ebd49e6b84fa0d37d90679f_866f4f82d2f771c80496e5b80211b690, store=[table=TestAcidGuarantees family=A region=866f4f82d2f771c80496e5b80211b690] 2024-11-27T07:27:06,233 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127bd40f8c31ebd49e6b84fa0d37d90679f_866f4f82d2f771c80496e5b80211b690 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=866f4f82d2f771c80496e5b80211b690] 2024-11-27T07:27:06,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742394_1570 (size=12561) 2024-11-27T07:27:06,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742395_1571 (size=4469) 2024-11-27T07:27:06,265 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 866f4f82d2f771c80496e5b80211b690#A#compaction#477 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:27:06,266 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/A/63fadd4517e04f2e8399d86b35c58db5 is 175, key is test_row_0/A:col10/1732692422621/Put/seqid=0 2024-11-27T07:27:06,269 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/B/f546472b7e014255ab9c968d9fa3c509 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/f546472b7e014255ab9c968d9fa3c509 2024-11-27T07:27:06,280 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 866f4f82d2f771c80496e5b80211b690/B of 866f4f82d2f771c80496e5b80211b690 into f546472b7e014255ab9c968d9fa3c509(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:27:06,280 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 866f4f82d2f771c80496e5b80211b690: 2024-11-27T07:27:06,280 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., storeName=866f4f82d2f771c80496e5b80211b690/B, priority=13, startTime=1732692426140; duration=0sec 2024-11-27T07:27:06,280 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:27:06,280 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 866f4f82d2f771c80496e5b80211b690:B 2024-11-27T07:27:06,280 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:27:06,282 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:27:06,282 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 866f4f82d2f771c80496e5b80211b690/C is initiating minor compaction (all files) 2024-11-27T07:27:06,282 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 866f4f82d2f771c80496e5b80211b690/C in TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:06,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742396_1572 (size=31515) 2024-11-27T07:27:06,283 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/34057b4b5fa14b1f9047b128bfcbd4ca, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/1cab719c6f6e49b0ab0ad8f65f2e766d, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/616de2be0dd74b7086c957ae21596669] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp, totalSize=35.9 K 2024-11-27T07:27:06,284 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 34057b4b5fa14b1f9047b128bfcbd4ca, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732692420491 2024-11-27T07:27:06,284 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 1cab719c6f6e49b0ab0ad8f65f2e766d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732692420865 2024-11-27T07:27:06,285 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 616de2be0dd74b7086c957ae21596669, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1732692422621 2024-11-27T07:27:06,289 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/A/63fadd4517e04f2e8399d86b35c58db5 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/63fadd4517e04f2e8399d86b35c58db5 2024-11-27T07:27:06,292 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-27T07:27:06,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:06,292 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 866f4f82d2f771c80496e5b80211b690 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-27T07:27:06,295 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 866f4f82d2f771c80496e5b80211b690#C#compaction#479 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:27:06,296 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/C/ac13c638985944189426d5b1deb9c6e0 is 50, key is test_row_0/C:col10/1732692422621/Put/seqid=0 2024-11-27T07:27:06,299 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 866f4f82d2f771c80496e5b80211b690/A of 866f4f82d2f771c80496e5b80211b690 into 63fadd4517e04f2e8399d86b35c58db5(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:27:06,299 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 866f4f82d2f771c80496e5b80211b690: 2024-11-27T07:27:06,299 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., storeName=866f4f82d2f771c80496e5b80211b690/A, priority=13, startTime=1732692426140; duration=0sec 2024-11-27T07:27:06,299 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:27:06,300 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 866f4f82d2f771c80496e5b80211b690:A 2024-11-27T07:27:06,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742397_1573 (size=12561) 2024-11-27T07:27:06,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127f207a0cef31349a1b85cbf4c65dc6d5a_866f4f82d2f771c80496e5b80211b690 is 50, key is test_row_0/A:col10/1732692424795/Put/seqid=0 2024-11-27T07:27:06,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742398_1574 (size=12304) 2024-11-27T07:27:06,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:27:06,768 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127f207a0cef31349a1b85cbf4c65dc6d5a_866f4f82d2f771c80496e5b80211b690 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127f207a0cef31349a1b85cbf4c65dc6d5a_866f4f82d2f771c80496e5b80211b690 2024-11-27T07:27:06,769 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/C/ac13c638985944189426d5b1deb9c6e0 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/ac13c638985944189426d5b1deb9c6e0 2024-11-27T07:27:06,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/A/d36fb6120d79433398807eb3ed89ec1b, store: [table=TestAcidGuarantees family=A region=866f4f82d2f771c80496e5b80211b690] 2024-11-27T07:27:06,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/A/d36fb6120d79433398807eb3ed89ec1b is 175, key is test_row_0/A:col10/1732692424795/Put/seqid=0 2024-11-27T07:27:06,772 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 866f4f82d2f771c80496e5b80211b690/C of 866f4f82d2f771c80496e5b80211b690 into ac13c638985944189426d5b1deb9c6e0(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:27:06,772 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 866f4f82d2f771c80496e5b80211b690: 2024-11-27T07:27:06,773 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., storeName=866f4f82d2f771c80496e5b80211b690/C, priority=13, startTime=1732692426140; duration=0sec 2024-11-27T07:27:06,773 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:27:06,773 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 866f4f82d2f771c80496e5b80211b690:C 2024-11-27T07:27:06,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742399_1575 (size=31105) 2024-11-27T07:27:06,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. as already flushing 2024-11-27T07:27:06,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 866f4f82d2f771c80496e5b80211b690 2024-11-27T07:27:06,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:06,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59888 deadline: 1732692486970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:06,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:06,973 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:06,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1732692486970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:06,973 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:06,973 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:06,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:06,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:06,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:06,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:06,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59810 deadline: 1732692486971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:06,974 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:06,974 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:06,974 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:07,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-27T07:27:07,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:07,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59888 deadline: 1732692487079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:07,079 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:07,080 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:07,080 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:07,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:07,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1732692487079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:07,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:07,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:07,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:07,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:07,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59810 deadline: 1732692487079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:07,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:07,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:07,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:07,174 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=195, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/A/d36fb6120d79433398807eb3ed89ec1b 2024-11-27T07:27:07,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/B/4c595bd420b24fd9bb13501c31554cac is 50, key is test_row_0/B:col10/1732692424795/Put/seqid=0 2024-11-27T07:27:07,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742400_1576 (size=12151) 2024-11-27T07:27:07,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:07,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59888 deadline: 1732692487289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:07,290 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:07,290 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:07,291 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:07,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:07,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1732692487289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:07,292 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:07,292 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:07,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:07,292 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:07,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59810 deadline: 1732692487289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:07,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:07,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:07,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:07,583 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/B/4c595bd420b24fd9bb13501c31554cac 2024-11-27T07:27:07,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/C/7215d5468cd14ae1ba1a21075600de87 is 50, key is test_row_0/C:col10/1732692424795/Put/seqid=0 2024-11-27T07:27:07,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742401_1577 (size=12151) 2024-11-27T07:27:07,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:07,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59888 deadline: 1732692487599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:07,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:07,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1732692487599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:07,601 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:07,601 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:07,601 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:07,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:07,601 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:07,602 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:07,602 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:07,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59810 deadline: 1732692487599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:07,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:07,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:07,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:07,992 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/C/7215d5468cd14ae1ba1a21075600de87 2024-11-27T07:27:07,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/A/d36fb6120d79433398807eb3ed89ec1b as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/d36fb6120d79433398807eb3ed89ec1b 2024-11-27T07:27:07,998 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/d36fb6120d79433398807eb3ed89ec1b, entries=150, sequenceid=195, filesize=30.4 K 2024-11-27T07:27:07,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/B/4c595bd420b24fd9bb13501c31554cac as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/4c595bd420b24fd9bb13501c31554cac 2024-11-27T07:27:08,001 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/4c595bd420b24fd9bb13501c31554cac, entries=150, sequenceid=195, filesize=11.9 K 2024-11-27T07:27:08,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/C/7215d5468cd14ae1ba1a21075600de87 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/7215d5468cd14ae1ba1a21075600de87 2024-11-27T07:27:08,004 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/7215d5468cd14ae1ba1a21075600de87, entries=150, sequenceid=195, filesize=11.9 K 2024-11-27T07:27:08,005 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 866f4f82d2f771c80496e5b80211b690 in 1713ms, sequenceid=195, compaction requested=false 2024-11-27T07:27:08,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 866f4f82d2f771c80496e5b80211b690: 2024-11-27T07:27:08,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:08,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-27T07:27:08,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-27T07:27:08,008 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-27T07:27:08,008 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1040 sec 2024-11-27T07:27:08,009 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 3.1080 sec 2024-11-27T07:27:08,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 866f4f82d2f771c80496e5b80211b690 2024-11-27T07:27:08,111 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T07:27:08,111 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 866f4f82d2f771c80496e5b80211b690 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-27T07:27:08,118 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112721eb391a6df54ffcad08911ad36debc0_866f4f82d2f771c80496e5b80211b690 is 50, key is test_row_0/A:col10/1732692426965/Put/seqid=0 2024-11-27T07:27:08,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742402_1578 (size=14794) 2024-11-27T07:27:08,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:08,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59810 deadline: 1732692488133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:08,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:08,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1732692488134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:08,137 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:08,137 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:08,137 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:08,137 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:08,137 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:08,137 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:08,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:08,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59888 deadline: 1732692488135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:08,138 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:08,138 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:08,138 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:08,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:08,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59810 deadline: 1732692488239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:08,241 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:08,241 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:08,241 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:08,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:08,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1732692488239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:08,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:08,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:08,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:08,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:08,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59888 deadline: 1732692488249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:08,251 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:08,251 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:08,251 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:08,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:08,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:08,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59810 deadline: 1732692488449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:08,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1732692488449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:08,452 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:08,452 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:08,452 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:08,452 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:08,452 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:08,452 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:08,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:08,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59888 deadline: 1732692488459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:08,460 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:08,460 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:08,460 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:08,521 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:27:08,524 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112721eb391a6df54ffcad08911ad36debc0_866f4f82d2f771c80496e5b80211b690 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112721eb391a6df54ffcad08911ad36debc0_866f4f82d2f771c80496e5b80211b690 2024-11-27T07:27:08,525 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/A/dd6d2bb2b72f472497df01a890b6b97d, store: [table=TestAcidGuarantees family=A region=866f4f82d2f771c80496e5b80211b690] 2024-11-27T07:27:08,526 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/A/dd6d2bb2b72f472497df01a890b6b97d is 175, key is test_row_0/A:col10/1732692426965/Put/seqid=0 2024-11-27T07:27:08,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742403_1579 (size=39749) 2024-11-27T07:27:08,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:08,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59810 deadline: 1732692488759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:08,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:08,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:08,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:08,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:08,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1732692488759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:08,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:08,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:08,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:08,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:08,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59888 deadline: 1732692488769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:08,770 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:08,770 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:08,770 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:08,930 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=216, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/A/dd6d2bb2b72f472497df01a890b6b97d 2024-11-27T07:27:08,935 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/B/088a90fbdee0438da19d477b2b62b369 is 50, key is test_row_0/B:col10/1732692426965/Put/seqid=0 2024-11-27T07:27:08,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742404_1580 (size=12151) 2024-11-27T07:27:09,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-27T07:27:09,070 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T07:27:09,070 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T07:27:09,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-27T07:27:09,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-27T07:27:09,072 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T07:27:09,072 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T07:27:09,072 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T07:27:09,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:09,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59808 deadline: 1732692489088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:09,091 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 289954c9202d,42117,1732692259671 for 'test_row_1' in 866f4f82d2f771c80496e5b80211b690 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T07:27:09,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:09,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:09,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:09,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-27T07:27:09,224 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-27T07:27:09,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:09,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. as already flushing 2024-11-27T07:27:09,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:09,224 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:27:09,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:27:09,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:27:09,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:09,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59810 deadline: 1732692489269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:09,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:09,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1732692489269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:09,272 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:09,272 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:09,272 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:09,272 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:09,272 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:09,272 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:09,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:09,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59888 deadline: 1732692489279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:09,282 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:09,282 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:09,282 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:09,338 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/B/088a90fbdee0438da19d477b2b62b369 2024-11-27T07:27:09,345 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/C/4dd53e55606843949233abcc023aff1a is 50, key is test_row_0/C:col10/1732692426965/Put/seqid=0 2024-11-27T07:27:09,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742405_1581 (size=12151) 2024-11-27T07:27:09,377 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-27T07:27:09,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:09,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. as already flushing 2024-11-27T07:27:09,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:09,377 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:27:09,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:27:09,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:27:09,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-27T07:27:09,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-27T07:27:09,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:09,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. as already flushing 2024-11-27T07:27:09,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:09,530 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:27:09,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:27:09,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:27:09,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-27T07:27:09,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:09,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. as already flushing 2024-11-27T07:27:09,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:09,683 ERROR [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:27:09,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:27:09,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T07:27:09,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-27T07:27:09,749 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/C/4dd53e55606843949233abcc023aff1a 2024-11-27T07:27:09,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/A/dd6d2bb2b72f472497df01a890b6b97d as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/dd6d2bb2b72f472497df01a890b6b97d 2024-11-27T07:27:09,755 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/dd6d2bb2b72f472497df01a890b6b97d, entries=200, sequenceid=216, filesize=38.8 K 2024-11-27T07:27:09,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/B/088a90fbdee0438da19d477b2b62b369 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/088a90fbdee0438da19d477b2b62b369 2024-11-27T07:27:09,758 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/088a90fbdee0438da19d477b2b62b369, entries=150, sequenceid=216, filesize=11.9 K 2024-11-27T07:27:09,758 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/C/4dd53e55606843949233abcc023aff1a as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/4dd53e55606843949233abcc023aff1a 2024-11-27T07:27:09,761 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/4dd53e55606843949233abcc023aff1a, entries=150, sequenceid=216, filesize=11.9 K 2024-11-27T07:27:09,761 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 866f4f82d2f771c80496e5b80211b690 in 1650ms, sequenceid=216, compaction requested=true 2024-11-27T07:27:09,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 866f4f82d2f771c80496e5b80211b690: 2024-11-27T07:27:09,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 866f4f82d2f771c80496e5b80211b690:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T07:27:09,762 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:27:09,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:27:09,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 866f4f82d2f771c80496e5b80211b690:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T07:27:09,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:27:09,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 866f4f82d2f771c80496e5b80211b690:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T07:27:09,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-27T07:27:09,762 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:27:09,762 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102369 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:27:09,762 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:27:09,763 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1541): 866f4f82d2f771c80496e5b80211b690/A is initiating minor compaction (all files) 2024-11-27T07:27:09,763 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 866f4f82d2f771c80496e5b80211b690/B is initiating minor compaction (all files) 2024-11-27T07:27:09,763 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 866f4f82d2f771c80496e5b80211b690/A in TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:09,763 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 866f4f82d2f771c80496e5b80211b690/B in TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:09,763 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/63fadd4517e04f2e8399d86b35c58db5, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/d36fb6120d79433398807eb3ed89ec1b, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/dd6d2bb2b72f472497df01a890b6b97d] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp, totalSize=100.0 K 2024-11-27T07:27:09,763 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/f546472b7e014255ab9c968d9fa3c509, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/4c595bd420b24fd9bb13501c31554cac, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/088a90fbdee0438da19d477b2b62b369] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp, totalSize=36.0 K 2024-11-27T07:27:09,763 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:09,763 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. files: [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/63fadd4517e04f2e8399d86b35c58db5, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/d36fb6120d79433398807eb3ed89ec1b, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/dd6d2bb2b72f472497df01a890b6b97d] 2024-11-27T07:27:09,763 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting f546472b7e014255ab9c968d9fa3c509, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1732692422621 2024-11-27T07:27:09,763 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63fadd4517e04f2e8399d86b35c58db5, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1732692422621 2024-11-27T07:27:09,763 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c595bd420b24fd9bb13501c31554cac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732692424795 2024-11-27T07:27:09,763 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting d36fb6120d79433398807eb3ed89ec1b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732692424795 2024-11-27T07:27:09,763 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 088a90fbdee0438da19d477b2b62b369, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732692426965 2024-11-27T07:27:09,763 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd6d2bb2b72f472497df01a890b6b97d, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732692426965 2024-11-27T07:27:09,767 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=866f4f82d2f771c80496e5b80211b690] 2024-11-27T07:27:09,768 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 866f4f82d2f771c80496e5b80211b690#B#compaction#486 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:27:09,769 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/B/9c34436075e34ba6ac3866fe8d609f44 is 50, key is test_row_0/B:col10/1732692426965/Put/seqid=0 2024-11-27T07:27:09,769 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127aa27250007fd4b7a96f56abd00a42e14_866f4f82d2f771c80496e5b80211b690 store=[table=TestAcidGuarantees family=A region=866f4f82d2f771c80496e5b80211b690] 2024-11-27T07:27:09,771 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127aa27250007fd4b7a96f56abd00a42e14_866f4f82d2f771c80496e5b80211b690, store=[table=TestAcidGuarantees family=A region=866f4f82d2f771c80496e5b80211b690] 2024-11-27T07:27:09,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742406_1582 (size=12663) 2024-11-27T07:27:09,771 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127aa27250007fd4b7a96f56abd00a42e14_866f4f82d2f771c80496e5b80211b690 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=866f4f82d2f771c80496e5b80211b690] 2024-11-27T07:27:09,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742407_1583 (size=4469) 2024-11-27T07:27:09,835 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-27T07:27:09,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:09,836 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 866f4f82d2f771c80496e5b80211b690 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-27T07:27:09,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127bbad1ecd95ec4205a300d58d2d4f1a7e_866f4f82d2f771c80496e5b80211b690 is 50, key is test_row_0/A:col10/1732692428128/Put/seqid=0 2024-11-27T07:27:09,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742408_1584 (size=12304) 2024-11-27T07:27:10,175 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 866f4f82d2f771c80496e5b80211b690#A#compaction#487 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:27:10,175 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/A/5bb1b3823e80472f9a6d37d03c740323 is 175, key is test_row_0/A:col10/1732692426965/Put/seqid=0 2024-11-27T07:27:10,176 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/B/9c34436075e34ba6ac3866fe8d609f44 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/9c34436075e34ba6ac3866fe8d609f44 2024-11-27T07:27:10,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742409_1585 (size=31617) 2024-11-27T07:27:10,180 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 866f4f82d2f771c80496e5b80211b690/B of 866f4f82d2f771c80496e5b80211b690 into 9c34436075e34ba6ac3866fe8d609f44(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:27:10,180 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 866f4f82d2f771c80496e5b80211b690: 2024-11-27T07:27:10,180 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., storeName=866f4f82d2f771c80496e5b80211b690/B, priority=13, startTime=1732692429762; duration=0sec 2024-11-27T07:27:10,180 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:27:10,180 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 866f4f82d2f771c80496e5b80211b690:B 2024-11-27T07:27:10,180 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T07:27:10,181 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T07:27:10,181 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 866f4f82d2f771c80496e5b80211b690/C is initiating minor compaction (all files) 2024-11-27T07:27:10,181 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 866f4f82d2f771c80496e5b80211b690/C in TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:10,181 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/ac13c638985944189426d5b1deb9c6e0, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/7215d5468cd14ae1ba1a21075600de87, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/4dd53e55606843949233abcc023aff1a] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp, totalSize=36.0 K 2024-11-27T07:27:10,182 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting ac13c638985944189426d5b1deb9c6e0, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1732692422621 2024-11-27T07:27:10,182 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 7215d5468cd14ae1ba1a21075600de87, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732692424795 2024-11-27T07:27:10,182 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 4dd53e55606843949233abcc023aff1a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732692426965 2024-11-27T07:27:10,187 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 866f4f82d2f771c80496e5b80211b690#C#compaction#489 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:27:10,187 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/C/2197ecc1fb4443b0870acc87ecaee713 is 50, key is test_row_0/C:col10/1732692426965/Put/seqid=0 2024-11-27T07:27:10,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742410_1586 (size=12663) 2024-11-27T07:27:10,193 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/C/2197ecc1fb4443b0870acc87ecaee713 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/2197ecc1fb4443b0870acc87ecaee713 2024-11-27T07:27:10,197 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 866f4f82d2f771c80496e5b80211b690/C of 866f4f82d2f771c80496e5b80211b690 into 2197ecc1fb4443b0870acc87ecaee713(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:27:10,197 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 866f4f82d2f771c80496e5b80211b690: 2024-11-27T07:27:10,197 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., storeName=866f4f82d2f771c80496e5b80211b690/C, priority=13, startTime=1732692429762; duration=0sec 2024-11-27T07:27:10,197 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:27:10,197 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 866f4f82d2f771c80496e5b80211b690:C 2024-11-27T07:27:10,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-27T07:27:10,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:27:10,248 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127bbad1ecd95ec4205a300d58d2d4f1a7e_866f4f82d2f771c80496e5b80211b690 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127bbad1ecd95ec4205a300d58d2d4f1a7e_866f4f82d2f771c80496e5b80211b690 2024-11-27T07:27:10,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/A/b8bfa048ae13456b80c82804e6c8e5f2, store: [table=TestAcidGuarantees family=A region=866f4f82d2f771c80496e5b80211b690] 2024-11-27T07:27:10,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/A/b8bfa048ae13456b80c82804e6c8e5f2 is 175, key is test_row_0/A:col10/1732692428128/Put/seqid=0 2024-11-27T07:27:10,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742411_1587 (size=31105) 2024-11-27T07:27:10,253 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=233, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/A/b8bfa048ae13456b80c82804e6c8e5f2 2024-11-27T07:27:10,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/B/10dc6b388321499a985b8397e174ab0a is 50, key is test_row_0/B:col10/1732692428128/Put/seqid=0 2024-11-27T07:27:10,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742412_1588 (size=12151) 2024-11-27T07:27:10,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 866f4f82d2f771c80496e5b80211b690 2024-11-27T07:27:10,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. as already flushing 2024-11-27T07:27:10,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:10,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59888 deadline: 1732692490312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:10,316 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:10,316 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:10,316 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:10,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:10,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59810 deadline: 1732692490314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:10,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:10,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:10,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1732692490314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:10,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:10,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:10,317 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:10,317 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:10,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:10,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:10,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59888 deadline: 1732692490419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:10,420 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:10,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:10,420 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:10,420 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:10,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59810 deadline: 1732692490419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:10,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:10,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:10,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:10,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:10,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1732692490429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:10,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:10,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:10,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:10,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:10,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59860 deadline: 1732692490539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:10,541 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 289954c9202d,42117,1732692259671 for 'test_row_0' in 866f4f82d2f771c80496e5b80211b690 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18268 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T07:27:10,542 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:10,542 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:10,542 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:10,582 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/A/5bb1b3823e80472f9a6d37d03c740323 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/5bb1b3823e80472f9a6d37d03c740323 2024-11-27T07:27:10,586 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 866f4f82d2f771c80496e5b80211b690/A of 866f4f82d2f771c80496e5b80211b690 into 5bb1b3823e80472f9a6d37d03c740323(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:27:10,586 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 866f4f82d2f771c80496e5b80211b690: 2024-11-27T07:27:10,586 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., storeName=866f4f82d2f771c80496e5b80211b690/A, priority=13, startTime=1732692429761; duration=0sec 2024-11-27T07:27:10,586 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:27:10,586 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 866f4f82d2f771c80496e5b80211b690:A 2024-11-27T07:27:10,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:10,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59888 deadline: 1732692490629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:10,631 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:10,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:10,631 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:10,631 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:10,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59810 deadline: 1732692490629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:10,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:10,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:10,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:10,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:10,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1732692490639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:10,641 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:10,641 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:10,641 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:10,661 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/B/10dc6b388321499a985b8397e174ab0a 2024-11-27T07:27:10,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/C/c906f419ec874d8aa46b0b2dcd95de87 is 50, key is test_row_0/C:col10/1732692428128/Put/seqid=0 2024-11-27T07:27:10,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742413_1589 (size=12151) 2024-11-27T07:27:10,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:10,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59888 deadline: 1732692490939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:10,941 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:10,941 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:10,941 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:10,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:10,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59810 deadline: 1732692490939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:10,942 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:10,942 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:10,942 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:10,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:10,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1732692490948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:10,951 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:10,951 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:10,951 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:11,069 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/C/c906f419ec874d8aa46b0b2dcd95de87 2024-11-27T07:27:11,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/A/b8bfa048ae13456b80c82804e6c8e5f2 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/b8bfa048ae13456b80c82804e6c8e5f2 2024-11-27T07:27:11,076 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/b8bfa048ae13456b80c82804e6c8e5f2, entries=150, sequenceid=233, filesize=30.4 K 2024-11-27T07:27:11,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/B/10dc6b388321499a985b8397e174ab0a as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/10dc6b388321499a985b8397e174ab0a 2024-11-27T07:27:11,079 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/10dc6b388321499a985b8397e174ab0a, entries=150, sequenceid=233, filesize=11.9 K 2024-11-27T07:27:11,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/C/c906f419ec874d8aa46b0b2dcd95de87 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/c906f419ec874d8aa46b0b2dcd95de87 2024-11-27T07:27:11,082 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/c906f419ec874d8aa46b0b2dcd95de87, entries=150, sequenceid=233, filesize=11.9 K 2024-11-27T07:27:11,083 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=114.05 KB/116790 for 866f4f82d2f771c80496e5b80211b690 in 1247ms, sequenceid=233, compaction requested=false 2024-11-27T07:27:11,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 866f4f82d2f771c80496e5b80211b690: 2024-11-27T07:27:11,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:11,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-27T07:27:11,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-27T07:27:11,086 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-27T07:27:11,086 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0120 sec 2024-11-27T07:27:11,087 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 2.0160 sec 2024-11-27T07:27:11,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-27T07:27:11,220 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T07:27:11,220 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-27T07:27:11,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-27T07:27:11,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-27T07:27:11,222 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T07:27:11,222 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T07:27:11,222 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T07:27:11,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-27T07:27:11,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42117 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-27T07:27:11,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:11,374 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 866f4f82d2f771c80496e5b80211b690 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-27T07:27:11,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411276b616aa171984a4498035d9f5eaffe9d_866f4f82d2f771c80496e5b80211b690 is 50, key is test_row_0/A:col10/1732692430312/Put/seqid=0 2024-11-27T07:27:11,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742414_1590 (size=12304) 2024-11-27T07:27:11,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 866f4f82d2f771c80496e5b80211b690 2024-11-27T07:27:11,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. as already flushing 2024-11-27T07:27:11,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:11,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1732692491468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:11,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:11,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:11,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59810 deadline: 1732692491468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:11,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:11,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:11,472 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:11,472 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:11,472 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:11,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:11,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59888 deadline: 1732692491472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:11,476 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:11,476 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:11,476 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:11,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-27T07:27:11,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:11,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1732692491579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:11,580 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:11,580 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:11,580 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:11,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:11,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59810 deadline: 1732692491579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:11,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:11,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:11,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:11,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:11,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59888 deadline: 1732692491579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:11,582 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:11,582 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:11,582 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:11,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:27:11,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:11,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1732692491788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:11,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:11,790 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:11,790 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:11,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59810 deadline: 1732692491789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:11,790 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:11,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:11,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:11,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:11,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59888 deadline: 1732692491789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:11,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:11,791 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:11,791 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:11,791 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:11,823 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411276b616aa171984a4498035d9f5eaffe9d_866f4f82d2f771c80496e5b80211b690 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411276b616aa171984a4498035d9f5eaffe9d_866f4f82d2f771c80496e5b80211b690 2024-11-27T07:27:11,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/A/57cf9e05c95247f69937d3b8d64c73b2, store: [table=TestAcidGuarantees family=A region=866f4f82d2f771c80496e5b80211b690] 2024-11-27T07:27:11,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/A/57cf9e05c95247f69937d3b8d64c73b2 is 175, key is test_row_0/A:col10/1732692430312/Put/seqid=0 2024-11-27T07:27:11,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742415_1591 (size=31105) 2024-11-27T07:27:11,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-27T07:27:12,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:12,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1732692492099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:12,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:12,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:12,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:12,102 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:12,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59810 deadline: 1732692492099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:12,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:12,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:12,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:12,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:12,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59888 deadline: 1732692492099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:12,102 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:12,102 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:12,102 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:12,227 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=256, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/A/57cf9e05c95247f69937d3b8d64c73b2 2024-11-27T07:27:12,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/B/a58bd367d52547189b377ee7367e759b is 50, key is test_row_0/B:col10/1732692430312/Put/seqid=0 2024-11-27T07:27:12,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742416_1592 (size=12151) 2024-11-27T07:27:12,250 INFO [Thread-2343 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2343. 2024-11-27T07:27:12,250 INFO [Thread-2341 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2341. 2024-11-27T07:27:12,250 DEBUG [Thread-2341 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T07:27:12,250 DEBUG [Thread-2343 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T07:27:12,250 DEBUG [Thread-2341 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:27:12,250 DEBUG [Thread-2343 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:27:12,250 DEBUG [Thread-2341 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:27:12,250 DEBUG [Thread-2343 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:27:12,250 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T07:27:12,250 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T07:27:12,251 INFO [Thread-2339 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2339. 2024-11-27T07:27:12,251 DEBUG [Thread-2339 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T07:27:12,251 DEBUG [Thread-2339 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:27:12,251 DEBUG [Thread-2339 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:27:12,251 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T07:27:12,253 INFO [Thread-2347 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2347. 2024-11-27T07:27:12,253 DEBUG [Thread-2347 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T07:27:12,253 DEBUG [Thread-2347 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:27:12,253 DEBUG [Thread-2347 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:27:12,253 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T07:27:12,254 INFO [Thread-2345 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2345. 2024-11-27T07:27:12,255 DEBUG [Thread-2345 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T07:27:12,255 DEBUG [Thread-2345 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:27:12,255 DEBUG [Thread-2345 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:27:12,255 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T07:27:12,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-27T07:27:12,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:12,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:12,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T07:27:12,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59840 deadline: 1732692492609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:12,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59810 deadline: 1732692492609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:12,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:59888 deadline: 1732692492609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 2024-11-27T07:27:12,610 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:12,610 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:12,610 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 , the old value is region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:12,610 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:12,610 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:12,610 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:12,610 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T07:27:12,610 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:12,610 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., hostname=289954c9202d,42117,1732692259671, seqNum=5 because the exception is null or not the one we care about 2024-11-27T07:27:12,636 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/B/a58bd367d52547189b377ee7367e759b 2024-11-27T07:27:12,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/C/0a8a3a0b55254f49a6c23b76c7ba3dd9 is 50, key is test_row_0/C:col10/1732692430312/Put/seqid=0 2024-11-27T07:27:12,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742417_1593 (size=12151) 2024-11-27T07:27:13,054 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/C/0a8a3a0b55254f49a6c23b76c7ba3dd9 2024-11-27T07:27:13,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/A/57cf9e05c95247f69937d3b8d64c73b2 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/57cf9e05c95247f69937d3b8d64c73b2 2024-11-27T07:27:13,069 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/57cf9e05c95247f69937d3b8d64c73b2, entries=150, sequenceid=256, filesize=30.4 K 2024-11-27T07:27:13,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/B/a58bd367d52547189b377ee7367e759b as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/a58bd367d52547189b377ee7367e759b 2024-11-27T07:27:13,074 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/a58bd367d52547189b377ee7367e759b, entries=150, sequenceid=256, filesize=11.9 K 2024-11-27T07:27:13,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/C/0a8a3a0b55254f49a6c23b76c7ba3dd9 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/0a8a3a0b55254f49a6c23b76c7ba3dd9 2024-11-27T07:27:13,079 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/0a8a3a0b55254f49a6c23b76c7ba3dd9, entries=150, sequenceid=256, filesize=11.9 K 2024-11-27T07:27:13,080 INFO [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 866f4f82d2f771c80496e5b80211b690 in 1706ms, sequenceid=256, compaction requested=true 2024-11-27T07:27:13,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 866f4f82d2f771c80496e5b80211b690: 2024-11-27T07:27:13,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:13,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/289954c9202d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-27T07:27:13,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-27T07:27:13,083 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-27T07:27:13,083 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8600 sec 2024-11-27T07:27:13,085 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 1.8640 sec 2024-11-27T07:27:13,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-27T07:27:13,370 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T07:27:13,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42117 {}] regionserver.HRegion(8855): Flush requested on 866f4f82d2f771c80496e5b80211b690 2024-11-27T07:27:13,622 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T07:27:13,622 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 866f4f82d2f771c80496e5b80211b690 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-27T07:27:13,622 INFO [Thread-2332 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2332. 2024-11-27T07:27:13,622 INFO [Thread-2336 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2336. 2024-11-27T07:27:13,622 INFO [Thread-2330 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2330. 2024-11-27T07:27:13,622 DEBUG [Thread-2336 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T07:27:13,622 DEBUG [Thread-2332 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T07:27:13,622 DEBUG [Thread-2336 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:27:13,622 DEBUG [Thread-2332 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:27:13,622 DEBUG [Thread-2330 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T07:27:13,622 DEBUG [Thread-2330 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:27:13,623 DEBUG [Thread-2332 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:27:13,623 DEBUG [Thread-2336 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:27:13,623 DEBUG [Thread-2330 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:27:13,623 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T07:27:13,623 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T07:27:13,623 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T07:27:13,630 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411271211b0b4a8784cc4987a33d3c2013826_866f4f82d2f771c80496e5b80211b690 is 50, key is test_row_0/A:col10/1732692433620/Put/seqid=0 2024-11-27T07:27:13,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742418_1594 (size=12454) 2024-11-27T07:27:13,633 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:27:13,636 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411271211b0b4a8784cc4987a33d3c2013826_866f4f82d2f771c80496e5b80211b690 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411271211b0b4a8784cc4987a33d3c2013826_866f4f82d2f771c80496e5b80211b690 2024-11-27T07:27:13,636 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/A/76895594d5974e0d8dd727229032f9ff, store: [table=TestAcidGuarantees family=A region=866f4f82d2f771c80496e5b80211b690] 2024-11-27T07:27:13,637 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/A/76895594d5974e0d8dd727229032f9ff is 175, key is test_row_0/A:col10/1732692433620/Put/seqid=0 2024-11-27T07:27:13,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742419_1595 (size=31255) 2024-11-27T07:27:14,041 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=275, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/A/76895594d5974e0d8dd727229032f9ff 2024-11-27T07:27:14,052 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/B/3f885bd4d3e34ca6aa98661e975a8d23 is 50, key is test_row_0/B:col10/1732692433620/Put/seqid=0 2024-11-27T07:27:14,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742420_1596 (size=12301) 2024-11-27T07:27:14,458 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/B/3f885bd4d3e34ca6aa98661e975a8d23 2024-11-27T07:27:14,473 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/C/ca2391f2d0544eebb7377137d726687a is 50, key is test_row_0/C:col10/1732692433620/Put/seqid=0 2024-11-27T07:27:14,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742421_1597 (size=12301) 2024-11-27T07:27:14,878 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/C/ca2391f2d0544eebb7377137d726687a 2024-11-27T07:27:14,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/A/76895594d5974e0d8dd727229032f9ff as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/76895594d5974e0d8dd727229032f9ff 2024-11-27T07:27:14,894 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/76895594d5974e0d8dd727229032f9ff, entries=150, sequenceid=275, filesize=30.5 K 2024-11-27T07:27:14,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/B/3f885bd4d3e34ca6aa98661e975a8d23 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/3f885bd4d3e34ca6aa98661e975a8d23 2024-11-27T07:27:14,898 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/3f885bd4d3e34ca6aa98661e975a8d23, entries=150, sequenceid=275, filesize=12.0 K 2024-11-27T07:27:14,899 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/C/ca2391f2d0544eebb7377137d726687a as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/ca2391f2d0544eebb7377137d726687a 2024-11-27T07:27:14,902 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/ca2391f2d0544eebb7377137d726687a, entries=150, sequenceid=275, filesize=12.0 K 2024-11-27T07:27:14,903 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=0 B/0 for 866f4f82d2f771c80496e5b80211b690 in 1281ms, sequenceid=275, compaction requested=true 2024-11-27T07:27:14,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 866f4f82d2f771c80496e5b80211b690: 2024-11-27T07:27:14,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 866f4f82d2f771c80496e5b80211b690:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T07:27:14,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:27:14,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 866f4f82d2f771c80496e5b80211b690:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T07:27:14,903 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T07:27:14,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:27:14,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 866f4f82d2f771c80496e5b80211b690:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T07:27:14,903 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T07:27:14,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:27:14,904 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49266 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T07:27:14,904 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 125082 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T07:27:14,905 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 866f4f82d2f771c80496e5b80211b690/B is initiating minor compaction (all files) 2024-11-27T07:27:14,905 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1541): 866f4f82d2f771c80496e5b80211b690/A is initiating minor compaction (all files) 2024-11-27T07:27:14,905 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 866f4f82d2f771c80496e5b80211b690/B in TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:14,905 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 866f4f82d2f771c80496e5b80211b690/A in TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:14,905 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/5bb1b3823e80472f9a6d37d03c740323, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/b8bfa048ae13456b80c82804e6c8e5f2, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/57cf9e05c95247f69937d3b8d64c73b2, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/76895594d5974e0d8dd727229032f9ff] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp, totalSize=122.2 K 2024-11-27T07:27:14,905 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/9c34436075e34ba6ac3866fe8d609f44, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/10dc6b388321499a985b8397e174ab0a, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/a58bd367d52547189b377ee7367e759b, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/3f885bd4d3e34ca6aa98661e975a8d23] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp, totalSize=48.1 K 2024-11-27T07:27:14,905 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:14,905 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. files: [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/5bb1b3823e80472f9a6d37d03c740323, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/b8bfa048ae13456b80c82804e6c8e5f2, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/57cf9e05c95247f69937d3b8d64c73b2, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/76895594d5974e0d8dd727229032f9ff] 2024-11-27T07:27:14,905 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c34436075e34ba6ac3866fe8d609f44, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732692426965 2024-11-27T07:27:14,905 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5bb1b3823e80472f9a6d37d03c740323, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732692426965 2024-11-27T07:27:14,905 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 10dc6b388321499a985b8397e174ab0a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732692428128 2024-11-27T07:27:14,906 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8bfa048ae13456b80c82804e6c8e5f2, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732692428128 2024-11-27T07:27:14,906 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting a58bd367d52547189b377ee7367e759b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732692430308 2024-11-27T07:27:14,906 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57cf9e05c95247f69937d3b8d64c73b2, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732692430308 2024-11-27T07:27:14,906 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f885bd4d3e34ca6aa98661e975a8d23, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732692431457 2024-11-27T07:27:14,906 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76895594d5974e0d8dd727229032f9ff, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732692431457 2024-11-27T07:27:14,913 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=866f4f82d2f771c80496e5b80211b690] 2024-11-27T07:27:14,914 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127db709c10cde14d3f8a59eae225485078_866f4f82d2f771c80496e5b80211b690 store=[table=TestAcidGuarantees family=A region=866f4f82d2f771c80496e5b80211b690] 2024-11-27T07:27:14,916 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 866f4f82d2f771c80496e5b80211b690#B#compaction#498 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:27:14,916 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/B/a96bd22b79be4b158cacdf0c09703858 is 50, key is test_row_0/B:col10/1732692433620/Put/seqid=0 2024-11-27T07:27:14,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742422_1598 (size=12949) 2024-11-27T07:27:14,922 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127db709c10cde14d3f8a59eae225485078_866f4f82d2f771c80496e5b80211b690, store=[table=TestAcidGuarantees family=A region=866f4f82d2f771c80496e5b80211b690] 2024-11-27T07:27:14,922 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127db709c10cde14d3f8a59eae225485078_866f4f82d2f771c80496e5b80211b690 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=866f4f82d2f771c80496e5b80211b690] 2024-11-27T07:27:14,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742423_1599 (size=4469) 2024-11-27T07:27:15,328 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 866f4f82d2f771c80496e5b80211b690#A#compaction#499 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:27:15,329 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/A/7fa46633c32d4479a3d81d56dd5038dd is 175, key is test_row_0/A:col10/1732692433620/Put/seqid=0 2024-11-27T07:27:15,330 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/B/a96bd22b79be4b158cacdf0c09703858 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/a96bd22b79be4b158cacdf0c09703858 2024-11-27T07:27:15,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742424_1600 (size=31903) 2024-11-27T07:27:15,335 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 866f4f82d2f771c80496e5b80211b690/B of 866f4f82d2f771c80496e5b80211b690 into a96bd22b79be4b158cacdf0c09703858(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:27:15,335 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 866f4f82d2f771c80496e5b80211b690: 2024-11-27T07:27:15,335 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., storeName=866f4f82d2f771c80496e5b80211b690/B, priority=12, startTime=1732692434903; duration=0sec 2024-11-27T07:27:15,335 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T07:27:15,335 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 866f4f82d2f771c80496e5b80211b690:B 2024-11-27T07:27:15,336 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T07:27:15,337 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49266 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T07:27:15,337 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1541): 866f4f82d2f771c80496e5b80211b690/C is initiating minor compaction (all files) 2024-11-27T07:27:15,337 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 866f4f82d2f771c80496e5b80211b690/C in TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:15,337 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/2197ecc1fb4443b0870acc87ecaee713, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/c906f419ec874d8aa46b0b2dcd95de87, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/0a8a3a0b55254f49a6c23b76c7ba3dd9, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/ca2391f2d0544eebb7377137d726687a] into tmpdir=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp, totalSize=48.1 K 2024-11-27T07:27:15,337 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 2197ecc1fb4443b0870acc87ecaee713, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732692426965 2024-11-27T07:27:15,338 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting c906f419ec874d8aa46b0b2dcd95de87, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732692428128 2024-11-27T07:27:15,338 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a8a3a0b55254f49a6c23b76c7ba3dd9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732692430308 2024-11-27T07:27:15,338 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] compactions.Compactor(225): Compacting ca2391f2d0544eebb7377137d726687a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732692431457 2024-11-27T07:27:15,349 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 866f4f82d2f771c80496e5b80211b690#C#compaction#500 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T07:27:15,350 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/C/5a03e231bf53440e893b1a2289097e7e is 50, key is test_row_0/C:col10/1732692433620/Put/seqid=0 2024-11-27T07:27:15,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742425_1601 (size=12949) 2024-11-27T07:27:15,742 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/A/7fa46633c32d4479a3d81d56dd5038dd as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/7fa46633c32d4479a3d81d56dd5038dd 2024-11-27T07:27:15,745 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 866f4f82d2f771c80496e5b80211b690/A of 866f4f82d2f771c80496e5b80211b690 into 7fa46633c32d4479a3d81d56dd5038dd(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:27:15,745 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 866f4f82d2f771c80496e5b80211b690: 2024-11-27T07:27:15,745 INFO [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., storeName=866f4f82d2f771c80496e5b80211b690/A, priority=12, startTime=1732692434903; duration=0sec 2024-11-27T07:27:15,746 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:27:15,746 DEBUG [RS:0;289954c9202d:42117-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 866f4f82d2f771c80496e5b80211b690:A 2024-11-27T07:27:15,759 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/C/5a03e231bf53440e893b1a2289097e7e as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/5a03e231bf53440e893b1a2289097e7e 2024-11-27T07:27:15,764 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 866f4f82d2f771c80496e5b80211b690/C of 866f4f82d2f771c80496e5b80211b690 into 5a03e231bf53440e893b1a2289097e7e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T07:27:15,764 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 866f4f82d2f771c80496e5b80211b690: 2024-11-27T07:27:15,764 INFO [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., storeName=866f4f82d2f771c80496e5b80211b690/C, priority=12, startTime=1732692434903; duration=0sec 2024-11-27T07:27:15,764 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T07:27:15,764 DEBUG [RS:0;289954c9202d:42117-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 866f4f82d2f771c80496e5b80211b690:C 2024-11-27T07:27:17,971 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-27T07:27:19,111 INFO [Thread-2334 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2334. 2024-11-27T07:27:19,111 DEBUG [Thread-2334 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T07:27:19,111 DEBUG [Thread-2334 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:27:19,111 DEBUG [Thread-2334 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:27:19,111 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T07:27:20,594 INFO [Thread-2328 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2328. 2024-11-27T07:27:20,594 DEBUG [Thread-2328 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T07:27:20,594 DEBUG [Thread-2328 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:27:20,594 DEBUG [Thread-2328 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:27:20,594 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T07:27:20,595 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-27T07:27:20,595 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 8 2024-11-27T07:27:20,595 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 60 2024-11-27T07:27:20,595 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 58 2024-11-27T07:27:20,595 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 38 2024-11-27T07:27:20,595 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-11-27T07:27:20,595 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-27T07:27:20,595 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4361 2024-11-27T07:27:20,595 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4288 2024-11-27T07:27:20,595 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4337 2024-11-27T07:27:20,595 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4358 2024-11-27T07:27:20,596 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4301 2024-11-27T07:27:20,596 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-27T07:27:20,596 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T07:27:20,596 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T07:27:20,596 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:27:20,597 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:27:20,597 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T07:27:20,598 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-27T07:27:20,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-27T07:27:20,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-27T07:27:20,602 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732692440602"}]},"ts":"1732692440602"} 2024-11-27T07:27:20,604 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-27T07:27:20,604 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-27T07:27:20,605 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-27T07:27:20,607 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=866f4f82d2f771c80496e5b80211b690, UNASSIGN}] 2024-11-27T07:27:20,608 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=866f4f82d2f771c80496e5b80211b690, UNASSIGN 2024-11-27T07:27:20,609 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=166 updating hbase:meta row=866f4f82d2f771c80496e5b80211b690, regionState=CLOSING, regionLocation=289954c9202d,42117,1732692259671 2024-11-27T07:27:20,610 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=866f4f82d2f771c80496e5b80211b690, UNASSIGN because future has completed 2024-11-27T07:27:20,610 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T07:27:20,611 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; CloseRegionProcedure 866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671}] 2024-11-27T07:27:20,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-27T07:27:20,764 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(122): Close 866f4f82d2f771c80496e5b80211b690 2024-11-27T07:27:20,764 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T07:27:20,765 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1722): Closing 866f4f82d2f771c80496e5b80211b690, disabling compactions & flushes 2024-11-27T07:27:20,765 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:20,765 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:20,765 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. after waiting 0 ms 2024-11-27T07:27:20,765 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:20,765 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(2902): Flushing 866f4f82d2f771c80496e5b80211b690 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-11-27T07:27:20,774 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127110ce77e71d543a4a738fcab62c21c14_866f4f82d2f771c80496e5b80211b690 is 50, key is test_row_0/A:col10/1732692440590/Put/seqid=0 2024-11-27T07:27:20,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742426_1602 (size=9914) 2024-11-27T07:27:20,779 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T07:27:20,783 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127110ce77e71d543a4a738fcab62c21c14_866f4f82d2f771c80496e5b80211b690 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127110ce77e71d543a4a738fcab62c21c14_866f4f82d2f771c80496e5b80211b690 2024-11-27T07:27:20,784 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/A/0717287ab0da419a91f822022beedacb, store: [table=TestAcidGuarantees family=A region=866f4f82d2f771c80496e5b80211b690] 2024-11-27T07:27:20,785 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/A/0717287ab0da419a91f822022beedacb is 175, key is test_row_0/A:col10/1732692440590/Put/seqid=0 2024-11-27T07:27:20,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742427_1603 (size=22561) 2024-11-27T07:27:20,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-27T07:27:21,190 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=283, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/A/0717287ab0da419a91f822022beedacb 2024-11-27T07:27:21,202 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/B/5abefcd7f7dc49639e6b3148d561bd22 is 50, key is test_row_0/B:col10/1732692440590/Put/seqid=0 2024-11-27T07:27:21,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742428_1604 (size=9857) 2024-11-27T07:27:21,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-27T07:27:21,609 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/B/5abefcd7f7dc49639e6b3148d561bd22 2024-11-27T07:27:21,620 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/C/e0e838b7a6874c0696878e21cd0c4b2d is 50, key is test_row_0/C:col10/1732692440590/Put/seqid=0 2024-11-27T07:27:21,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742429_1605 (size=9857) 2024-11-27T07:27:21,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-27T07:27:22,024 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/C/e0e838b7a6874c0696878e21cd0c4b2d 2024-11-27T07:27:22,030 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/A/0717287ab0da419a91f822022beedacb as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/0717287ab0da419a91f822022beedacb 2024-11-27T07:27:22,035 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/0717287ab0da419a91f822022beedacb, entries=100, sequenceid=283, filesize=22.0 K 2024-11-27T07:27:22,036 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/B/5abefcd7f7dc49639e6b3148d561bd22 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/5abefcd7f7dc49639e6b3148d561bd22 2024-11-27T07:27:22,040 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/5abefcd7f7dc49639e6b3148d561bd22, entries=100, sequenceid=283, filesize=9.6 K 2024-11-27T07:27:22,041 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/.tmp/C/e0e838b7a6874c0696878e21cd0c4b2d as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/e0e838b7a6874c0696878e21cd0c4b2d 2024-11-27T07:27:22,044 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/e0e838b7a6874c0696878e21cd0c4b2d, entries=100, sequenceid=283, filesize=9.6 K 2024-11-27T07:27:22,045 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 866f4f82d2f771c80496e5b80211b690 in 1280ms, sequenceid=283, compaction requested=false 2024-11-27T07:27:22,046 DEBUG [StoreCloser-TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/9477368a26614e42bf26c794bf89a48e, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/8e76bc29d89d4a09acbc4f2b271d3699, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/de09385b04c94ac89d8307ed809d5079, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/915da50deffb479aaff10b836ed2b8c5, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/511ad2efdd144292acda8ca66f11fa52, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/2cb3afa99a6b431d8075d24acff213d0, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/eaa36318d26742adbeb0ffc775c6c069, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/e06dd5d0cc544e699e9eb27948f1e155, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/64e3b5df06f14459b1777e302fa41f7f, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/5029d8a2d3714ade96b87b46210a241f, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/70a9d5eeb86748ea85e80847ade7a471, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/63fadd4517e04f2e8399d86b35c58db5, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/c6bb39fe6d88419c82352c33b9f2a8d3, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/d36fb6120d79433398807eb3ed89ec1b, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/dd6d2bb2b72f472497df01a890b6b97d, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/5bb1b3823e80472f9a6d37d03c740323, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/b8bfa048ae13456b80c82804e6c8e5f2, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/57cf9e05c95247f69937d3b8d64c73b2, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/76895594d5974e0d8dd727229032f9ff] to archive 2024-11-27T07:27:22,047 DEBUG [StoreCloser-TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T07:27:22,048 DEBUG [StoreCloser-TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/9477368a26614e42bf26c794bf89a48e to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/9477368a26614e42bf26c794bf89a48e 2024-11-27T07:27:22,050 DEBUG [StoreCloser-TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/8e76bc29d89d4a09acbc4f2b271d3699 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/8e76bc29d89d4a09acbc4f2b271d3699 2024-11-27T07:27:22,051 DEBUG [StoreCloser-TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/de09385b04c94ac89d8307ed809d5079 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/de09385b04c94ac89d8307ed809d5079 2024-11-27T07:27:22,053 DEBUG [StoreCloser-TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/915da50deffb479aaff10b836ed2b8c5 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/915da50deffb479aaff10b836ed2b8c5 2024-11-27T07:27:22,054 DEBUG [StoreCloser-TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/511ad2efdd144292acda8ca66f11fa52 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/511ad2efdd144292acda8ca66f11fa52 2024-11-27T07:27:22,055 DEBUG [StoreCloser-TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/2cb3afa99a6b431d8075d24acff213d0 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/2cb3afa99a6b431d8075d24acff213d0 2024-11-27T07:27:22,057 DEBUG [StoreCloser-TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/eaa36318d26742adbeb0ffc775c6c069 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/eaa36318d26742adbeb0ffc775c6c069 2024-11-27T07:27:22,058 DEBUG [StoreCloser-TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/e06dd5d0cc544e699e9eb27948f1e155 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/e06dd5d0cc544e699e9eb27948f1e155 2024-11-27T07:27:22,058 DEBUG [StoreCloser-TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/64e3b5df06f14459b1777e302fa41f7f to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/64e3b5df06f14459b1777e302fa41f7f 2024-11-27T07:27:22,059 DEBUG [StoreCloser-TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/5029d8a2d3714ade96b87b46210a241f to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/5029d8a2d3714ade96b87b46210a241f 2024-11-27T07:27:22,060 DEBUG [StoreCloser-TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/70a9d5eeb86748ea85e80847ade7a471 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/70a9d5eeb86748ea85e80847ade7a471 2024-11-27T07:27:22,061 DEBUG [StoreCloser-TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/63fadd4517e04f2e8399d86b35c58db5 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/63fadd4517e04f2e8399d86b35c58db5 2024-11-27T07:27:22,061 DEBUG [StoreCloser-TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/c6bb39fe6d88419c82352c33b9f2a8d3 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/c6bb39fe6d88419c82352c33b9f2a8d3 2024-11-27T07:27:22,062 DEBUG [StoreCloser-TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/d36fb6120d79433398807eb3ed89ec1b to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/d36fb6120d79433398807eb3ed89ec1b 2024-11-27T07:27:22,063 DEBUG [StoreCloser-TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/dd6d2bb2b72f472497df01a890b6b97d to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/dd6d2bb2b72f472497df01a890b6b97d 2024-11-27T07:27:22,063 DEBUG [StoreCloser-TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/5bb1b3823e80472f9a6d37d03c740323 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/5bb1b3823e80472f9a6d37d03c740323 2024-11-27T07:27:22,064 DEBUG [StoreCloser-TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/b8bfa048ae13456b80c82804e6c8e5f2 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/b8bfa048ae13456b80c82804e6c8e5f2 2024-11-27T07:27:22,065 DEBUG [StoreCloser-TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/57cf9e05c95247f69937d3b8d64c73b2 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/57cf9e05c95247f69937d3b8d64c73b2 2024-11-27T07:27:22,065 DEBUG [StoreCloser-TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/76895594d5974e0d8dd727229032f9ff to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/76895594d5974e0d8dd727229032f9ff 2024-11-27T07:27:22,066 DEBUG [StoreCloser-TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/47c724e907e14a61865d440ef649fdd1, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/62b5a08980c544bfa2b98228395dbaaa, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/cafd58d3855f4cafbdc1aedff133a749, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/6dc3b53b69aa45c6bc6f5099b3cfd915, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/e135b9e32b9e4d9da3217b5eff77a71b, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/cbaab1e139f947d4a24e68e1e3d387a3, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/09e806b8cbb049ddb1a0599a8cb3e161, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/757e19a53aaa4b5cb1e00a06f6d83dfd, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/e11af2ddf64e420692cca2bfa9b88a47, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/e547e6b683204b51b63f48bd5b9d767c, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/0c5c8049f7cf4a259df080f194eb4cc0, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/f546472b7e014255ab9c968d9fa3c509, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/c2b926b158714d33b221d97f29e4c6f7, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/4c595bd420b24fd9bb13501c31554cac, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/9c34436075e34ba6ac3866fe8d609f44, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/088a90fbdee0438da19d477b2b62b369, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/10dc6b388321499a985b8397e174ab0a, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/a58bd367d52547189b377ee7367e759b, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/3f885bd4d3e34ca6aa98661e975a8d23] to archive 2024-11-27T07:27:22,067 DEBUG [StoreCloser-TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T07:27:22,068 DEBUG [StoreCloser-TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/47c724e907e14a61865d440ef649fdd1 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/47c724e907e14a61865d440ef649fdd1 2024-11-27T07:27:22,068 DEBUG [StoreCloser-TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/62b5a08980c544bfa2b98228395dbaaa to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/62b5a08980c544bfa2b98228395dbaaa 2024-11-27T07:27:22,069 DEBUG [StoreCloser-TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/cafd58d3855f4cafbdc1aedff133a749 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/cafd58d3855f4cafbdc1aedff133a749 2024-11-27T07:27:22,070 DEBUG [StoreCloser-TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/6dc3b53b69aa45c6bc6f5099b3cfd915 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/6dc3b53b69aa45c6bc6f5099b3cfd915 2024-11-27T07:27:22,070 DEBUG [StoreCloser-TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/e135b9e32b9e4d9da3217b5eff77a71b to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/e135b9e32b9e4d9da3217b5eff77a71b 2024-11-27T07:27:22,071 DEBUG [StoreCloser-TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/cbaab1e139f947d4a24e68e1e3d387a3 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/cbaab1e139f947d4a24e68e1e3d387a3 2024-11-27T07:27:22,072 DEBUG [StoreCloser-TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/09e806b8cbb049ddb1a0599a8cb3e161 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/09e806b8cbb049ddb1a0599a8cb3e161 2024-11-27T07:27:22,072 DEBUG [StoreCloser-TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/757e19a53aaa4b5cb1e00a06f6d83dfd to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/757e19a53aaa4b5cb1e00a06f6d83dfd 2024-11-27T07:27:22,073 DEBUG [StoreCloser-TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/e11af2ddf64e420692cca2bfa9b88a47 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/e11af2ddf64e420692cca2bfa9b88a47 2024-11-27T07:27:22,074 DEBUG [StoreCloser-TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/e547e6b683204b51b63f48bd5b9d767c to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/e547e6b683204b51b63f48bd5b9d767c 2024-11-27T07:27:22,075 DEBUG [StoreCloser-TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/0c5c8049f7cf4a259df080f194eb4cc0 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/0c5c8049f7cf4a259df080f194eb4cc0 2024-11-27T07:27:22,075 DEBUG [StoreCloser-TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/f546472b7e014255ab9c968d9fa3c509 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/f546472b7e014255ab9c968d9fa3c509 2024-11-27T07:27:22,076 DEBUG [StoreCloser-TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/c2b926b158714d33b221d97f29e4c6f7 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/c2b926b158714d33b221d97f29e4c6f7 2024-11-27T07:27:22,077 DEBUG [StoreCloser-TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/4c595bd420b24fd9bb13501c31554cac to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/4c595bd420b24fd9bb13501c31554cac 2024-11-27T07:27:22,077 DEBUG [StoreCloser-TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/9c34436075e34ba6ac3866fe8d609f44 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/9c34436075e34ba6ac3866fe8d609f44 2024-11-27T07:27:22,078 DEBUG [StoreCloser-TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/088a90fbdee0438da19d477b2b62b369 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/088a90fbdee0438da19d477b2b62b369 2024-11-27T07:27:22,078 DEBUG [StoreCloser-TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/10dc6b388321499a985b8397e174ab0a to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/10dc6b388321499a985b8397e174ab0a 2024-11-27T07:27:22,079 DEBUG [StoreCloser-TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/a58bd367d52547189b377ee7367e759b to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/a58bd367d52547189b377ee7367e759b 2024-11-27T07:27:22,080 DEBUG [StoreCloser-TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/3f885bd4d3e34ca6aa98661e975a8d23 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/3f885bd4d3e34ca6aa98661e975a8d23 2024-11-27T07:27:22,081 DEBUG [StoreCloser-TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/90063e32a3444686b89317add3fabd8e, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/399218b9e6e34ceca97f948d02a64294, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/282ae5b7e80b4f3dbb982d842789eb9b, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/89d1b340afdf4b5fa26093f52d2f109b, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/5574eb832c3e4184bcb55c991d6d34da, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/61bddf22f49844bb81de3b8c71f890d9, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/03cb44524dfb4f448891625b5112b4b1, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/74d6142d0abe4be8854b9790405fad78, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/34057b4b5fa14b1f9047b128bfcbd4ca, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/c969d12d2b1a4d1b95d3c2111ab4f92b, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/1cab719c6f6e49b0ab0ad8f65f2e766d, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/ac13c638985944189426d5b1deb9c6e0, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/616de2be0dd74b7086c957ae21596669, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/7215d5468cd14ae1ba1a21075600de87, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/2197ecc1fb4443b0870acc87ecaee713, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/4dd53e55606843949233abcc023aff1a, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/c906f419ec874d8aa46b0b2dcd95de87, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/0a8a3a0b55254f49a6c23b76c7ba3dd9, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/ca2391f2d0544eebb7377137d726687a] to archive 2024-11-27T07:27:22,081 DEBUG [StoreCloser-TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T07:27:22,082 DEBUG [StoreCloser-TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/90063e32a3444686b89317add3fabd8e to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/90063e32a3444686b89317add3fabd8e 2024-11-27T07:27:22,083 DEBUG [StoreCloser-TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/399218b9e6e34ceca97f948d02a64294 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/399218b9e6e34ceca97f948d02a64294 2024-11-27T07:27:22,083 DEBUG [StoreCloser-TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/282ae5b7e80b4f3dbb982d842789eb9b to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/282ae5b7e80b4f3dbb982d842789eb9b 2024-11-27T07:27:22,084 DEBUG [StoreCloser-TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/89d1b340afdf4b5fa26093f52d2f109b to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/89d1b340afdf4b5fa26093f52d2f109b 2024-11-27T07:27:22,085 DEBUG [StoreCloser-TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/5574eb832c3e4184bcb55c991d6d34da to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/5574eb832c3e4184bcb55c991d6d34da 2024-11-27T07:27:22,085 DEBUG [StoreCloser-TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/61bddf22f49844bb81de3b8c71f890d9 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/61bddf22f49844bb81de3b8c71f890d9 2024-11-27T07:27:22,086 DEBUG [StoreCloser-TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/03cb44524dfb4f448891625b5112b4b1 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/03cb44524dfb4f448891625b5112b4b1 2024-11-27T07:27:22,087 DEBUG [StoreCloser-TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/74d6142d0abe4be8854b9790405fad78 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/74d6142d0abe4be8854b9790405fad78 2024-11-27T07:27:22,087 DEBUG [StoreCloser-TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/34057b4b5fa14b1f9047b128bfcbd4ca to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/34057b4b5fa14b1f9047b128bfcbd4ca 2024-11-27T07:27:22,088 DEBUG [StoreCloser-TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/c969d12d2b1a4d1b95d3c2111ab4f92b to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/c969d12d2b1a4d1b95d3c2111ab4f92b 2024-11-27T07:27:22,088 DEBUG [StoreCloser-TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/1cab719c6f6e49b0ab0ad8f65f2e766d to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/1cab719c6f6e49b0ab0ad8f65f2e766d 2024-11-27T07:27:22,089 DEBUG [StoreCloser-TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/ac13c638985944189426d5b1deb9c6e0 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/ac13c638985944189426d5b1deb9c6e0 2024-11-27T07:27:22,089 DEBUG [StoreCloser-TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/616de2be0dd74b7086c957ae21596669 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/616de2be0dd74b7086c957ae21596669 2024-11-27T07:27:22,090 DEBUG [StoreCloser-TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/7215d5468cd14ae1ba1a21075600de87 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/7215d5468cd14ae1ba1a21075600de87 2024-11-27T07:27:22,091 DEBUG [StoreCloser-TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/2197ecc1fb4443b0870acc87ecaee713 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/2197ecc1fb4443b0870acc87ecaee713 2024-11-27T07:27:22,091 DEBUG [StoreCloser-TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/4dd53e55606843949233abcc023aff1a to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/4dd53e55606843949233abcc023aff1a 2024-11-27T07:27:22,092 DEBUG [StoreCloser-TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/c906f419ec874d8aa46b0b2dcd95de87 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/c906f419ec874d8aa46b0b2dcd95de87 2024-11-27T07:27:22,092 DEBUG [StoreCloser-TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/0a8a3a0b55254f49a6c23b76c7ba3dd9 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/0a8a3a0b55254f49a6c23b76c7ba3dd9 2024-11-27T07:27:22,093 DEBUG [StoreCloser-TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/ca2391f2d0544eebb7377137d726687a to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/ca2391f2d0544eebb7377137d726687a 2024-11-27T07:27:22,096 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/recovered.edits/286.seqid, newMaxSeqId=286, maxSeqId=4 2024-11-27T07:27:22,096 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. 2024-11-27T07:27:22,096 DEBUG [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1676): Region close journal for 866f4f82d2f771c80496e5b80211b690: Waiting for close lock at 1732692440764Running coprocessor pre-close hooks at 1732692440764Disabling compacts and flushes for region at 1732692440765 (+1 ms)Disabling writes for close at 1732692440765Obtaining lock to block concurrent updates at 1732692440765Preparing flush snapshotting stores in 866f4f82d2f771c80496e5b80211b690 at 1732692440765Finished memstore snapshotting TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=36720, getOffHeapSize=0, getCellsCount=300 at 1732692440766 (+1 ms)Flushing stores of TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690. at 1732692440767 (+1 ms)Flushing 866f4f82d2f771c80496e5b80211b690/A: creating writer at 1732692440768 (+1 ms)Flushing mob file 866f4f82d2f771c80496e5b80211b690/A: appending metadata at 1732692440774 (+6 ms)Flushing mob file 866f4f82d2f771c80496e5b80211b690/A: closing flushed file at 1732692440774Flushing 866f4f82d2f771c80496e5b80211b690/A: appending metadata at 1732692440784 (+10 ms)Flushing 866f4f82d2f771c80496e5b80211b690/A: closing flushed file at 1732692440784Flushing 866f4f82d2f771c80496e5b80211b690/B: creating writer at 1732692441198 (+414 ms)Flushing 866f4f82d2f771c80496e5b80211b690/B: appending metadata at 1732692441201 (+3 ms)Flushing 866f4f82d2f771c80496e5b80211b690/B: closing flushed file at 1732692441201Flushing 866f4f82d2f771c80496e5b80211b690/C: creating writer at 1732692441617 (+416 ms)Flushing 866f4f82d2f771c80496e5b80211b690/C: appending metadata at 1732692441620 (+3 ms)Flushing 866f4f82d2f771c80496e5b80211b690/C: closing flushed file at 1732692441620Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7bd1b291: reopening flushed file at 1732692442029 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@779bd923: reopening flushed file at 1732692442035 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1b0f8d23: reopening flushed file at 1732692442040 (+5 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 866f4f82d2f771c80496e5b80211b690 in 1280ms, sequenceid=283, compaction requested=false at 1732692442045 (+5 ms)Writing region close event to WAL at 1732692442094 (+49 ms)Running coprocessor post-close hooks at 1732692442096 (+2 ms)Closed at 1732692442096 2024-11-27T07:27:22,098 INFO [RS_CLOSE_REGION-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(157): Closed 866f4f82d2f771c80496e5b80211b690 2024-11-27T07:27:22,098 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=166 updating hbase:meta row=866f4f82d2f771c80496e5b80211b690, regionState=CLOSED 2024-11-27T07:27:22,099 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=167, ppid=166, state=RUNNABLE, hasLock=false; CloseRegionProcedure 866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 because future has completed 2024-11-27T07:27:22,101 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-27T07:27:22,101 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; CloseRegionProcedure 866f4f82d2f771c80496e5b80211b690, server=289954c9202d,42117,1732692259671 in 1.4900 sec 2024-11-27T07:27:22,102 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=166, resume processing ppid=165 2024-11-27T07:27:22,102 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, ppid=165, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=866f4f82d2f771c80496e5b80211b690, UNASSIGN in 1.4940 sec 2024-11-27T07:27:22,104 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-27T07:27:22,104 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4980 sec 2024-11-27T07:27:22,105 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732692442105"}]},"ts":"1732692442105"} 2024-11-27T07:27:22,106 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-27T07:27:22,106 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-27T07:27:22,107 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5080 sec 2024-11-27T07:27:22,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-27T07:27:22,751 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-27T07:27:22,752 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-27T07:27:22,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T07:27:22,757 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=168, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T07:27:22,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-27T07:27:22,758 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=168, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T07:27:22,761 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690 2024-11-27T07:27:22,763 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A, FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B, FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C, FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/recovered.edits] 2024-11-27T07:27:22,765 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/0717287ab0da419a91f822022beedacb to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/0717287ab0da419a91f822022beedacb 2024-11-27T07:27:22,766 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/7fa46633c32d4479a3d81d56dd5038dd to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/A/7fa46633c32d4479a3d81d56dd5038dd 2024-11-27T07:27:22,767 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/5abefcd7f7dc49639e6b3148d561bd22 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/5abefcd7f7dc49639e6b3148d561bd22 2024-11-27T07:27:22,768 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/a96bd22b79be4b158cacdf0c09703858 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/B/a96bd22b79be4b158cacdf0c09703858 2024-11-27T07:27:22,770 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/5a03e231bf53440e893b1a2289097e7e to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/5a03e231bf53440e893b1a2289097e7e 2024-11-27T07:27:22,771 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/e0e838b7a6874c0696878e21cd0c4b2d to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/C/e0e838b7a6874c0696878e21cd0c4b2d 2024-11-27T07:27:22,772 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/recovered.edits/286.seqid to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690/recovered.edits/286.seqid 2024-11-27T07:27:22,773 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/default/TestAcidGuarantees/866f4f82d2f771c80496e5b80211b690 2024-11-27T07:27:22,773 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-27T07:27:22,773 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-27T07:27:22,774 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-27T07:27:22,775 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127110ce77e71d543a4a738fcab62c21c14_866f4f82d2f771c80496e5b80211b690 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127110ce77e71d543a4a738fcab62c21c14_866f4f82d2f771c80496e5b80211b690 2024-11-27T07:27:22,776 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411271211b0b4a8784cc4987a33d3c2013826_866f4f82d2f771c80496e5b80211b690 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411271211b0b4a8784cc4987a33d3c2013826_866f4f82d2f771c80496e5b80211b690 2024-11-27T07:27:22,777 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112721eb391a6df54ffcad08911ad36debc0_866f4f82d2f771c80496e5b80211b690 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112721eb391a6df54ffcad08911ad36debc0_866f4f82d2f771c80496e5b80211b690 2024-11-27T07:27:22,778 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411272992cfca95424557bcc1041bb6b64b99_866f4f82d2f771c80496e5b80211b690 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411272992cfca95424557bcc1041bb6b64b99_866f4f82d2f771c80496e5b80211b690 2024-11-27T07:27:22,779 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112736a8806fa6074db3ab700809c2dc743b_866f4f82d2f771c80496e5b80211b690 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112736a8806fa6074db3ab700809c2dc743b_866f4f82d2f771c80496e5b80211b690 2024-11-27T07:27:22,779 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127382ed256c44f4ab4af59457cc5de1273_866f4f82d2f771c80496e5b80211b690 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127382ed256c44f4ab4af59457cc5de1273_866f4f82d2f771c80496e5b80211b690 2024-11-27T07:27:22,780 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112752a672fba7b54d0ab52695fc679e37cd_866f4f82d2f771c80496e5b80211b690 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112752a672fba7b54d0ab52695fc679e37cd_866f4f82d2f771c80496e5b80211b690 2024-11-27T07:27:22,781 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112753edc7f41e86414c882c0a0d4f2b4cc6_866f4f82d2f771c80496e5b80211b690 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112753edc7f41e86414c882c0a0d4f2b4cc6_866f4f82d2f771c80496e5b80211b690 2024-11-27T07:27:22,782 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127624982cd006641c2a181106fde3ef266_866f4f82d2f771c80496e5b80211b690 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127624982cd006641c2a181106fde3ef266_866f4f82d2f771c80496e5b80211b690 2024-11-27T07:27:22,782 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411276b616aa171984a4498035d9f5eaffe9d_866f4f82d2f771c80496e5b80211b690 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411276b616aa171984a4498035d9f5eaffe9d_866f4f82d2f771c80496e5b80211b690 2024-11-27T07:27:22,783 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411278a6c0f83f88b4ccb9a436646b0ba667b_866f4f82d2f771c80496e5b80211b690 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411278a6c0f83f88b4ccb9a436646b0ba667b_866f4f82d2f771c80496e5b80211b690 2024-11-27T07:27:22,784 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411278e0b6996bf26410991d786d74ebc408d_866f4f82d2f771c80496e5b80211b690 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411278e0b6996bf26410991d786d74ebc408d_866f4f82d2f771c80496e5b80211b690 2024-11-27T07:27:22,785 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127bbad1ecd95ec4205a300d58d2d4f1a7e_866f4f82d2f771c80496e5b80211b690 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127bbad1ecd95ec4205a300d58d2d4f1a7e_866f4f82d2f771c80496e5b80211b690 2024-11-27T07:27:22,786 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127f207a0cef31349a1b85cbf4c65dc6d5a_866f4f82d2f771c80496e5b80211b690 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127f207a0cef31349a1b85cbf4c65dc6d5a_866f4f82d2f771c80496e5b80211b690 2024-11-27T07:27:22,787 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127fa197227c584409aa1cfb9b14cb69645_866f4f82d2f771c80496e5b80211b690 to hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127fa197227c584409aa1cfb9b14cb69645_866f4f82d2f771c80496e5b80211b690 2024-11-27T07:27:22,787 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-27T07:27:22,789 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=168, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T07:27:22,792 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-27T07:27:22,794 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-27T07:27:22,795 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=168, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T07:27:22,795 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-27T07:27:22,795 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732692442795"}]},"ts":"9223372036854775807"} 2024-11-27T07:27:22,797 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-27T07:27:22,797 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 866f4f82d2f771c80496e5b80211b690, NAME => 'TestAcidGuarantees,,1732692409392.866f4f82d2f771c80496e5b80211b690.', STARTKEY => '', ENDKEY => ''}] 2024-11-27T07:27:22,797 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-27T07:27:22,797 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732692442797"}]},"ts":"9223372036854775807"} 2024-11-27T07:27:22,799 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-27T07:27:22,800 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=168, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T07:27:22,801 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 48 msec 2024-11-27T07:27:22,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43057 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-27T07:27:22,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-27T07:27:22,875 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-27T07:27:22,887 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=237 (was 240), OpenFileDescriptor=449 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=343 (was 446), ProcessCount=11 (was 11), AvailableMemoryMB=4388 (was 4474) 2024-11-27T07:27:22,888 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-27T07:27:22,888 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T07:27:22,888 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T07:27:22,888 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:27:22,888 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:27:22,888 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T07:27:22,888 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-27T07:27:22,888 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1219616701, stopped=false 2024-11-27T07:27:22,889 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=289954c9202d,43057,1732692258941 2024-11-27T07:27:23,016 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43057-0x1017b2da8960000, quorum=127.0.0.1:51145, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-27T07:27:23,016 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42117-0x1017b2da8960001, quorum=127.0.0.1:51145, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-27T07:27:23,016 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43057-0x1017b2da8960000, quorum=127.0.0.1:51145, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T07:27:23,016 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42117-0x1017b2da8960001, quorum=127.0.0.1:51145, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T07:27:23,016 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T07:27:23,017 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42117-0x1017b2da8960001, quorum=127.0.0.1:51145, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T07:27:23,017 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43057-0x1017b2da8960000, quorum=127.0.0.1:51145, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T07:27:23,017 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T07:27:23,017 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T07:27:23,017 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:27:23,018 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '289954c9202d,42117,1732692259671' ***** 2024-11-27T07:27:23,018 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-27T07:27:23,018 INFO [RS:0;289954c9202d:42117 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-27T07:27:23,018 INFO [RS:0;289954c9202d:42117 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-27T07:27:23,018 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-27T07:27:23,018 INFO [RS:0;289954c9202d:42117 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-27T07:27:23,018 INFO [RS:0;289954c9202d:42117 {}] regionserver.HRegionServer(959): stopping server 289954c9202d,42117,1732692259671 2024-11-27T07:27:23,018 INFO [RS:0;289954c9202d:42117 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-27T07:27:23,018 INFO [RS:0;289954c9202d:42117 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;289954c9202d:42117. 2024-11-27T07:27:23,018 DEBUG [RS:0;289954c9202d:42117 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T07:27:23,019 DEBUG [RS:0;289954c9202d:42117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:27:23,019 INFO [RS:0;289954c9202d:42117 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-27T07:27:23,019 INFO [RS:0;289954c9202d:42117 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-27T07:27:23,019 INFO [RS:0;289954c9202d:42117 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-27T07:27:23,019 INFO [RS:0;289954c9202d:42117 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-27T07:27:23,019 INFO [RS:0;289954c9202d:42117 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-27T07:27:23,020 DEBUG [RS:0;289954c9202d:42117 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-27T07:27:23,020 DEBUG [RS:0;289954c9202d:42117 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-27T07:27:23,020 DEBUG [RS_CLOSE_META-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-27T07:27:23,020 INFO [RS_CLOSE_META-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-27T07:27:23,020 DEBUG [RS_CLOSE_META-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-27T07:27:23,020 DEBUG [RS_CLOSE_META-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-27T07:27:23,020 DEBUG [RS_CLOSE_META-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-27T07:27:23,020 INFO [RS_CLOSE_META-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-27T07:27:23,040 DEBUG [RS_CLOSE_META-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/hbase/meta/1588230740/.tmp/info/67329e928f2c44daa781e90b7135d259 is 95, key is TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851./info:/1732692291756/DeleteFamily/seqid=0 2024-11-27T07:27:23,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742430_1606 (size=6514) 2024-11-27T07:27:23,062 INFO [regionserver/289954c9202d:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T07:27:23,220 DEBUG [RS:0;289954c9202d:42117 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-27T07:27:23,420 DEBUG [RS:0;289954c9202d:42117 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-27T07:27:23,443 INFO [RS_CLOSE_META-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/hbase/meta/1588230740/.tmp/info/67329e928f2c44daa781e90b7135d259 2024-11-27T07:27:23,470 DEBUG [RS_CLOSE_META-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/hbase/meta/1588230740/.tmp/ns/786dd2f89a3c4c5abb2bb662fbcc1cd4 is 93, key is TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851./ns:/1732692291705/DeleteFamily/seqid=0 2024-11-27T07:27:23,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742431_1607 (size=5906) 2024-11-27T07:27:23,621 DEBUG [RS:0;289954c9202d:42117 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-27T07:27:23,821 DEBUG [RS:0;289954c9202d:42117 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-27T07:27:23,874 INFO [RS_CLOSE_META-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/hbase/meta/1588230740/.tmp/ns/786dd2f89a3c4c5abb2bb662fbcc1cd4 2024-11-27T07:27:23,899 DEBUG [RS_CLOSE_META-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/hbase/meta/1588230740/.tmp/rep_barrier/878317476ec44624859fec4987a9b97b is 102, key is TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851./rep_barrier:/1732692291705/DeleteFamily/seqid=0 2024-11-27T07:27:23,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742432_1608 (size=6025) 2024-11-27T07:27:23,988 INFO [regionserver/289954c9202d:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-27T07:27:23,988 INFO [regionserver/289954c9202d:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-27T07:27:24,021 DEBUG [RS:0;289954c9202d:42117 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-27T07:27:24,222 DEBUG [RS:0;289954c9202d:42117 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-27T07:27:24,304 INFO [RS_CLOSE_META-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/hbase/meta/1588230740/.tmp/rep_barrier/878317476ec44624859fec4987a9b97b 2024-11-27T07:27:24,329 DEBUG [RS_CLOSE_META-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/hbase/meta/1588230740/.tmp/table/854e473fe93c4e27b4c52d7651f952d9 is 96, key is TestAcidGuarantees,,1732692263665.18c429ab939f89ba8ff6429b6bcb0851./table:/1732692291705/DeleteFamily/seqid=0 2024-11-27T07:27:24,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742433_1609 (size=5936) 2024-11-27T07:27:24,422 DEBUG [RS:0;289954c9202d:42117 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-27T07:27:24,622 DEBUG [RS:0;289954c9202d:42117 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-27T07:27:24,733 INFO [RS_CLOSE_META-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/hbase/meta/1588230740/.tmp/table/854e473fe93c4e27b4c52d7651f952d9 2024-11-27T07:27:24,739 DEBUG [RS_CLOSE_META-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/hbase/meta/1588230740/.tmp/info/67329e928f2c44daa781e90b7135d259 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/hbase/meta/1588230740/info/67329e928f2c44daa781e90b7135d259 2024-11-27T07:27:24,743 INFO [RS_CLOSE_META-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/hbase/meta/1588230740/info/67329e928f2c44daa781e90b7135d259, entries=12, sequenceid=90, filesize=6.4 K 2024-11-27T07:27:24,744 DEBUG [RS_CLOSE_META-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/hbase/meta/1588230740/.tmp/ns/786dd2f89a3c4c5abb2bb662fbcc1cd4 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/hbase/meta/1588230740/ns/786dd2f89a3c4c5abb2bb662fbcc1cd4 2024-11-27T07:27:24,748 INFO [RS_CLOSE_META-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/hbase/meta/1588230740/ns/786dd2f89a3c4c5abb2bb662fbcc1cd4, entries=8, sequenceid=90, filesize=5.8 K 2024-11-27T07:27:24,748 DEBUG [RS_CLOSE_META-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/hbase/meta/1588230740/.tmp/rep_barrier/878317476ec44624859fec4987a9b97b as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/hbase/meta/1588230740/rep_barrier/878317476ec44624859fec4987a9b97b 2024-11-27T07:27:24,752 INFO [RS_CLOSE_META-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/hbase/meta/1588230740/rep_barrier/878317476ec44624859fec4987a9b97b, entries=6, sequenceid=90, filesize=5.9 K 2024-11-27T07:27:24,752 DEBUG [RS_CLOSE_META-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/hbase/meta/1588230740/.tmp/table/854e473fe93c4e27b4c52d7651f952d9 as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/hbase/meta/1588230740/table/854e473fe93c4e27b4c52d7651f952d9 2024-11-27T07:27:24,755 INFO [RS_CLOSE_META-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/hbase/meta/1588230740/table/854e473fe93c4e27b4c52d7651f952d9, entries=7, sequenceid=90, filesize=5.8 K 2024-11-27T07:27:24,756 INFO [RS_CLOSE_META-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1736ms, sequenceid=90, compaction requested=false 2024-11-27T07:27:24,762 DEBUG [RS_CLOSE_META-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-27T07:27:24,762 DEBUG [RS_CLOSE_META-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-27T07:27:24,762 INFO [RS_CLOSE_META-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-27T07:27:24,762 DEBUG [RS_CLOSE_META-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732692443019Running coprocessor pre-close hooks at 1732692443020 (+1 ms)Disabling compacts and flushes for region at 1732692443020Disabling writes for close at 1732692443020Obtaining lock to block concurrent updates at 1732692443020Preparing flush snapshotting stores in 1588230740 at 1732692443020Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1732692443020Flushing stores of hbase:meta,,1.1588230740 at 1732692443021 (+1 ms)Flushing 1588230740/info: creating writer at 1732692443021Flushing 1588230740/info: appending metadata at 1732692443039 (+18 ms)Flushing 1588230740/info: closing flushed file at 1732692443039Flushing 1588230740/ns: creating writer at 1732692443450 (+411 ms)Flushing 1588230740/ns: appending metadata at 1732692443469 (+19 ms)Flushing 1588230740/ns: closing flushed file at 1732692443469Flushing 1588230740/rep_barrier: creating writer at 1732692443883 (+414 ms)Flushing 1588230740/rep_barrier: appending metadata at 1732692443899 (+16 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1732692443899Flushing 1588230740/table: creating writer at 1732692444311 (+412 ms)Flushing 1588230740/table: appending metadata at 1732692444329 (+18 ms)Flushing 1588230740/table: closing flushed file at 1732692444329Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@a283765: reopening flushed file at 1732692444738 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@522d420e: reopening flushed file at 1732692444743 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@71907080: reopening flushed file at 1732692444748 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@29155621: reopening flushed file at 1732692444752 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1736ms, sequenceid=90, compaction requested=false at 1732692444756 (+4 ms)Writing region close event to WAL at 1732692444758 (+2 ms)Running coprocessor post-close hooks at 1732692444762 (+4 ms)Closed at 1732692444762 2024-11-27T07:27:24,763 DEBUG [RS_CLOSE_META-regionserver/289954c9202d:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-27T07:27:24,823 INFO [RS:0;289954c9202d:42117 {}] regionserver.HRegionServer(976): stopping server 289954c9202d,42117,1732692259671; all regions closed. 2024-11-27T07:27:24,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741834_1010 (size=25344) 2024-11-27T07:27:24,836 DEBUG [RS:0;289954c9202d:42117 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/oldWALs 2024-11-27T07:27:24,836 INFO [RS:0;289954c9202d:42117 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 289954c9202d%2C42117%2C1732692259671.meta:.meta(num 1732692263167) 2024-11-27T07:27:24,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741832_1008 (size=13373750) 2024-11-27T07:27:24,841 DEBUG [RS:0;289954c9202d:42117 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/oldWALs 2024-11-27T07:27:24,841 INFO [RS:0;289954c9202d:42117 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 289954c9202d%2C42117%2C1732692259671:(num 1732692262112) 2024-11-27T07:27:24,841 DEBUG [RS:0;289954c9202d:42117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T07:27:24,841 INFO [RS:0;289954c9202d:42117 {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T07:27:24,841 INFO [RS:0;289954c9202d:42117 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-27T07:27:24,842 INFO [RS:0;289954c9202d:42117 {}] hbase.ChoreService(370): Chore service for: regionserver/289954c9202d:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-27T07:27:24,842 INFO [RS:0;289954c9202d:42117 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-27T07:27:24,842 INFO [regionserver/289954c9202d:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-27T07:27:24,843 INFO [RS:0;289954c9202d:42117 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:42117 2024-11-27T07:27:24,904 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42117-0x1017b2da8960001, quorum=127.0.0.1:51145, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/289954c9202d,42117,1732692259671 2024-11-27T07:27:24,904 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43057-0x1017b2da8960000, quorum=127.0.0.1:51145, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-27T07:27:24,904 INFO [RS:0;289954c9202d:42117 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-27T07:27:24,915 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [289954c9202d,42117,1732692259671] 2024-11-27T07:27:24,924 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/289954c9202d,42117,1732692259671 already deleted, retry=false 2024-11-27T07:27:24,925 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 289954c9202d,42117,1732692259671 expired; onlineServers=0 2024-11-27T07:27:24,925 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '289954c9202d,43057,1732692258941' ***** 2024-11-27T07:27:24,925 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-27T07:27:24,925 INFO [M:0;289954c9202d:43057 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-27T07:27:24,925 INFO [M:0;289954c9202d:43057 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-27T07:27:24,925 DEBUG [M:0;289954c9202d:43057 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-27T07:27:24,925 DEBUG [M:0;289954c9202d:43057 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-27T07:27:24,925 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-27T07:27:24,926 DEBUG [master/289954c9202d:0:becomeActiveMaster-HFileCleaner.small.0-1732692261814 {}] cleaner.HFileCleaner(306): Exit Thread[master/289954c9202d:0:becomeActiveMaster-HFileCleaner.small.0-1732692261814,5,FailOnTimeoutGroup] 2024-11-27T07:27:24,926 DEBUG [master/289954c9202d:0:becomeActiveMaster-HFileCleaner.large.0-1732692261813 {}] cleaner.HFileCleaner(306): Exit Thread[master/289954c9202d:0:becomeActiveMaster-HFileCleaner.large.0-1732692261813,5,FailOnTimeoutGroup] 2024-11-27T07:27:24,926 INFO [M:0;289954c9202d:43057 {}] hbase.ChoreService(370): Chore service for: master/289954c9202d:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-27T07:27:24,926 INFO [M:0;289954c9202d:43057 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-27T07:27:24,926 DEBUG [M:0;289954c9202d:43057 {}] master.HMaster(1795): Stopping service threads 2024-11-27T07:27:24,926 INFO [M:0;289954c9202d:43057 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-27T07:27:24,926 INFO [M:0;289954c9202d:43057 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T07:27:24,927 ERROR [M:0;289954c9202d:43057 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-27T07:27:24,928 INFO [M:0;289954c9202d:43057 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-27T07:27:24,928 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-27T07:27:24,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43057-0x1017b2da8960000, quorum=127.0.0.1:51145, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-27T07:27:24,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43057-0x1017b2da8960000, quorum=127.0.0.1:51145, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T07:27:24,936 DEBUG [M:0;289954c9202d:43057 {}] zookeeper.ZKUtil(347): master:43057-0x1017b2da8960000, quorum=127.0.0.1:51145, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-27T07:27:24,936 WARN [M:0;289954c9202d:43057 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-27T07:27:24,937 INFO [M:0;289954c9202d:43057 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/.lastflushedseqids 2024-11-27T07:27:24,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742434_1610 (size=72) 2024-11-27T07:27:25,015 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42117-0x1017b2da8960001, quorum=127.0.0.1:51145, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T07:27:25,015 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42117-0x1017b2da8960001, quorum=127.0.0.1:51145, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T07:27:25,016 INFO [RS:0;289954c9202d:42117 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-27T07:27:25,016 INFO [RS:0;289954c9202d:42117 {}] regionserver.HRegionServer(1031): Exiting; stopping=289954c9202d,42117,1732692259671; zookeeper connection closed. 2024-11-27T07:27:25,017 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6691c863 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6691c863 2024-11-27T07:27:25,018 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-27T07:27:25,349 INFO [M:0;289954c9202d:43057 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-27T07:27:25,349 INFO [M:0;289954c9202d:43057 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-27T07:27:25,350 DEBUG [M:0;289954c9202d:43057 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-27T07:27:25,350 INFO [M:0;289954c9202d:43057 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T07:27:25,350 DEBUG [M:0;289954c9202d:43057 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T07:27:25,350 DEBUG [M:0;289954c9202d:43057 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-27T07:27:25,350 DEBUG [M:0;289954c9202d:43057 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T07:27:25,350 INFO [M:0;289954c9202d:43057 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=750.92 KB heapSize=922.89 KB 2024-11-27T07:27:25,363 DEBUG [M:0;289954c9202d:43057 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/51cc9eebb1304bf791edb2fffec8158a is 82, key is hbase:meta,,1/info:regioninfo/1732692263251/Put/seqid=0 2024-11-27T07:27:25,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742435_1611 (size=5672) 2024-11-27T07:27:25,768 INFO [M:0;289954c9202d:43057 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2124 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/51cc9eebb1304bf791edb2fffec8158a 2024-11-27T07:27:25,799 DEBUG [M:0;289954c9202d:43057 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a10cbe1467f243e49f30932f317db22c is 2276, key is \x00\x00\x00\x00\x00\x00\x00\x91/proc:d/1732692412206/Put/seqid=0 2024-11-27T07:27:25,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742436_1612 (size=43767) 2024-11-27T07:27:26,203 INFO [M:0;289954c9202d:43057 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750.37 KB at sequenceid=2124 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a10cbe1467f243e49f30932f317db22c 2024-11-27T07:27:26,208 INFO [M:0;289954c9202d:43057 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a10cbe1467f243e49f30932f317db22c 2024-11-27T07:27:26,227 DEBUG [M:0;289954c9202d:43057 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/55f28abee5e24750b00cd7ae4d2a59fc is 69, key is 289954c9202d,42117,1732692259671/rs:state/1732692261869/Put/seqid=0 2024-11-27T07:27:26,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073742437_1613 (size=5156) 2024-11-27T07:27:26,632 INFO [M:0;289954c9202d:43057 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2124 (bloomFilter=true), to=hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/55f28abee5e24750b00cd7ae4d2a59fc 2024-11-27T07:27:26,641 DEBUG [M:0;289954c9202d:43057 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/51cc9eebb1304bf791edb2fffec8158a as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/51cc9eebb1304bf791edb2fffec8158a 2024-11-27T07:27:26,648 INFO [M:0;289954c9202d:43057 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/51cc9eebb1304bf791edb2fffec8158a, entries=8, sequenceid=2124, filesize=5.5 K 2024-11-27T07:27:26,649 DEBUG [M:0;289954c9202d:43057 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a10cbe1467f243e49f30932f317db22c as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a10cbe1467f243e49f30932f317db22c 2024-11-27T07:27:26,652 INFO [M:0;289954c9202d:43057 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a10cbe1467f243e49f30932f317db22c 2024-11-27T07:27:26,652 INFO [M:0;289954c9202d:43057 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a10cbe1467f243e49f30932f317db22c, entries=168, sequenceid=2124, filesize=42.7 K 2024-11-27T07:27:26,653 DEBUG [M:0;289954c9202d:43057 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/55f28abee5e24750b00cd7ae4d2a59fc as hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/55f28abee5e24750b00cd7ae4d2a59fc 2024-11-27T07:27:26,656 INFO [M:0;289954c9202d:43057 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39369/user/jenkins/test-data/37f85e4d-a0f6-0176-c5cc-3860318c64b4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/55f28abee5e24750b00cd7ae4d2a59fc, entries=1, sequenceid=2124, filesize=5.0 K 2024-11-27T07:27:26,657 INFO [M:0;289954c9202d:43057 {}] regionserver.HRegion(3140): Finished flush of dataSize ~750.92 KB/768944, heapSize ~922.59 KB/944736, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1307ms, sequenceid=2124, compaction requested=false 2024-11-27T07:27:26,658 INFO [M:0;289954c9202d:43057 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T07:27:26,658 DEBUG [M:0;289954c9202d:43057 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732692445350Disabling compacts and flushes for region at 1732692445350Disabling writes for close at 1732692445350Obtaining lock to block concurrent updates at 1732692445350Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732692445350Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=768944, getHeapSize=944976, getOffHeapSize=0, getCellsCount=2317 at 1732692445350Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732692445350Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732692445351 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732692445363 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732692445363Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732692445775 (+412 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732692445798 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732692445798Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732692446208 (+410 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732692446226 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732692446226Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@79b2414d: reopening flushed file at 1732692446640 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@289ea192: reopening flushed file at 1732692446648 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1fdc2d31: reopening flushed file at 1732692446652 (+4 ms)Finished flush of dataSize ~750.92 KB/768944, heapSize ~922.59 KB/944736, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1307ms, sequenceid=2124, compaction requested=false at 1732692446657 (+5 ms)Writing region close event to WAL at 1732692446658 (+1 ms)Closed at 1732692446658 2024-11-27T07:27:26,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44229 is added to blk_1073741830_1006 (size=908195) 2024-11-27T07:27:26,661 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-27T07:27:26,661 INFO [M:0;289954c9202d:43057 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-27T07:27:26,661 INFO [M:0;289954c9202d:43057 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:43057 2024-11-27T07:27:26,662 INFO [M:0;289954c9202d:43057 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-27T07:27:26,815 INFO [M:0;289954c9202d:43057 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-27T07:27:26,815 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43057-0x1017b2da8960000, quorum=127.0.0.1:51145, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T07:27:26,815 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43057-0x1017b2da8960000, quorum=127.0.0.1:51145, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T07:27:26,825 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2c68b1bf{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-27T07:27:26,827 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3efe07f8{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-27T07:27:26,827 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-27T07:27:26,828 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@42946cd0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-27T07:27:26,828 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@336e9627{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b939589-5539-0f0f-ceb6-48662b1efadc/hadoop.log.dir/,STOPPED} 2024-11-27T07:27:26,831 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-27T07:27:26,831 WARN [BP-336076723-172.17.0.3-1732692254946 heartbeating to localhost/127.0.0.1:39369 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-27T07:27:26,831 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-27T07:27:26,831 WARN [BP-336076723-172.17.0.3-1732692254946 heartbeating to localhost/127.0.0.1:39369 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-336076723-172.17.0.3-1732692254946 (Datanode Uuid 96c860c6-dc59-478e-903a-a301ba3afd05) service to localhost/127.0.0.1:39369 2024-11-27T07:27:26,835 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b939589-5539-0f0f-ceb6-48662b1efadc/cluster_8a8651c6-cd4e-11be-2ecd-ca6a64beeda5/data/data1/current/BP-336076723-172.17.0.3-1732692254946 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-27T07:27:26,835 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b939589-5539-0f0f-ceb6-48662b1efadc/cluster_8a8651c6-cd4e-11be-2ecd-ca6a64beeda5/data/data2/current/BP-336076723-172.17.0.3-1732692254946 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-27T07:27:26,836 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-27T07:27:26,843 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-27T07:27:26,844 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-27T07:27:26,844 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-27T07:27:26,844 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-27T07:27:26,844 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b939589-5539-0f0f-ceb6-48662b1efadc/hadoop.log.dir/,STOPPED} 2024-11-27T07:27:26,860 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-27T07:27:26,981 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down